def dotest(outputname, nostamp):
try:
from bidi.algorithm import get_display
except ImportError:
from unittest import SkipTest
raise SkipTest("Need python-bidi")
pdf = FPDF()
if nostamp:
pdf._putinfo = lambda: common.test_putinfo(pdf)
pdf.compress = False
pdf.add_page()
pdf.add_font('DejaVu', '', \
os.path.join(common.basepath, 'font/DejaVuSans.ttf'), uni=True)
pdf.set_font('DejaVu', '', 14)
# this will be displayed wrong as actually it is stored LTR:
text= u"این یک متن پارسی است. This is a Persian text !!"
pdf.write(8, text)
pdf.ln(8)
# Reverse the RLT using the Bidirectional Algorithm to be displayed correctly:
# (http://unicode.org/reports/tr9/)
rtl_text = get_display(text)
pdf.write(8, rtl_text)
pdf.output(outputname, 'F')
def test_explicit_with_upper_is_rtl(self):
"""Explicit tests"""
tests = (
(u"this is _LJUST_o", u"this is JUST"),
(u"a _lsimple _RteST_o th_oat", u"a simple TSet that"),
(u"HAS A _LPDF missing", u"PDF missing A SAH"),
(u"AnD hOw_L AbOuT, 123,987 tHiS_o", u"w AbOuT, 123,987 tHiSOh DnA"),
(u"a GOOD - _L_oTEST.", u"a TSET - DOOG."),
(u"here_L is_o_o_o _R a good one_o", u"here is eno doog a"),
(u"THE _rbest _lONE and", u"best ENO and EHT"),
(u"A REAL BIG_l_o BUG!", u"!GUB GIB LAER A"),
(u"a _L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_Rbug", u"a gub"),
(u"AN ARABIC _l_o 123-456 NICE ONE!", u"!ENO ECIN 456-123 CIBARA NA"),
(u"AN ARABIC _l _o 123-456 PAIR", u"RIAP 123-456 CIBARA NA"),
(u"this bug 67_r_o89 catched!", u"this bug 6789 catched!"),
)
# adopt fribidi's CapRtl encoding
mappings = {
u"_>": u"\u200E",
u"_<": u"\u200F",
u"_l": u"\u202A",
u"_r": u"\u202B",
u"_o": u"\u202C",
u"_L": u"\u202D",
u"_R": u"\u202E",
u"__": "_",
}
for storage, display in tests:
for key, val in mappings.items():
storage = storage.replace(key, val)
self.assertEqual(get_display(storage, upper_is_rtl=True), display)
def test_explicit_with_upper_is_rtl(self):
"""Explicit tests"""
tests = (
(u'this is _LJUST_o', u'this is JUST'),
(u'a _lsimple _RteST_o th_oat', u'a simple TSet that'),
(u'HAS A _LPDF missing', u'PDF missing A SAH'),
(u'AnD hOw_L AbOuT, 123,987 tHiS_o', u'w AbOuT, 123,987 tHiSOh DnA'),
(u'a GOOD - _L_oTEST.', u'a TSET - DOOG.'),
(u'here_L is_o_o_o _R a good one_o', u'here is eno doog a'),
(u'THE _rbest _lONE and', u'best ENO and EHT'),
(u'A REAL BIG_l_o BUG!', u'!GUB GIB LAER A'),
(u'a _L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_L_Rbug', u'a gub'),
(u'AN ARABIC _l_o 123-456 NICE ONE!', u'!ENO ECIN 456-123 CIBARA NA'),
(u'AN ARABIC _l _o 123-456 PAIR', u'RIAP 123-456 CIBARA NA'),
(u'this bug 67_r_o89 catched!', u'this bug 6789 catched!'),
)
# adopt fribidi's CapRtl encoding
mappings = {
u'_>': u"\u200E",
u'_<': u"\u200F",
u'_l': u"\u202A",
u'_r': u"\u202B",
u'_o': u"\u202C",
u'_L': u"\u202D",
u'_R': u"\u202E",
u'__': '_',
}
for storage, display in tests:
for key, val in mappings.items():
storage = storage.replace(key, val)
self.assertEqual(get_display(storage, upper_is_rtl=True), display)
def drawText(canvas, x, y, text, en = False, bold = False, size = 12):
wrkText = text
isArabic = False
isBidi = False
for c in wrkText:
cat = unicodedata.bidirectional(c)
if cat == "AL" or cat == "AN" or cat == "FA":
isArabic = True
isBidi = True
break
elif cat == "R" or cat == "RLE" or cat == "RLO":
isBidi = True
if isArabic:
wrkText = a_forms.fuse(wrkText)
wrkText = a_process.shape(wrkText)
if isBidi:
wrkText = get_display(wrkText)
if bold:
canvas.setFont('BNazanin', size)
else:
canvas.setFont('Nazanin', size)
canvas.drawRightString(x, canvas._pagesize[1] - y, wrkText)
def test_output_encoding(self):
"""Make sure the display is in the same encdoing as the incoming text"""
storage = "\xf9\xec\xe5\xed" # Hebrew word shalom in cp1255
display = "\xed\xe5\xec\xf9"
self.assertEqual(get_display(storage, encoding="cp1255"), display)
def test_implict_with_upper_is_rtl(self):
'''Implicit tests'''
tests = (
(u'car is THE CAR in arabic', u'car is RAC EHT in arabic'),
(u'CAR IS the car IN ENGLISH', u'HSILGNE NI the car SI RAC'),
(u'he said "IT IS 123, 456, OK"', u'he said "KO ,456 ,123 SI TI"'),
(u'he said "IT IS (123, 456), OK"',
u'he said "KO ,(456 ,123) SI TI"'),
(u'he said "IT IS 123,456, OK"', u'he said "KO ,123,456 SI TI"'),
(u'he said "IT IS (123,456), OK"',
u'he said "KO ,(123,456) SI TI"'),
(u'HE SAID "it is 123, 456, ok"', u'"it is 123, 456, ok" DIAS EH'),
(u'<H123>shalom</H123>', u'<123H/>shalom<123H>'),
(u'<h123>SAALAM</h123>', u'<h123>MALAAS</h123>'),
(u'HE SAID "it is a car!" AND RAN',
u'NAR DNA "!it is a car" DIAS EH'),
(u'HE SAID "it is a car!x" AND RAN',
u'NAR DNA "it is a car!x" DIAS EH'),
(u'SOLVE 1*5 1-5 1/5 1+5', u'1+5 1/5 1-5 5*1 EVLOS'),
(u'THE RANGE IS 2.5..5', u'5..2.5 SI EGNAR EHT'),
(u'-2 CELSIUS IS COLD', u'DLOC SI SUISLEC 2-'),
)
for storage, display in tests:
self.assertEqual(get_display(storage, upper_is_rtl=True), display)
def render_text(self, text, lang_text, base_font_size, bold_font,
pos_x, pos_y, transform=True):
if transform:
text = self.get_text(text, lang_text)
base_font = is_base(text)
offset = 0
for line in text.splitlines():
# Iterate until text fits into widget
for font_size in range(base_font_size, 3, -1):
font = get_font(font_size, bold_font, base_font)
layout_size = font.getsize(line)
layout_width = layout_size[0]
if layout_width + pos_x < self.width:
break
# Render text
self.draw.text(
(pos_x, pos_y + offset),
get_display(line),
font=font,
fill=COLOR_DATA[self.color]['text']
)
offset += layout_size[1]
def test_output_encoding(self):
"""Make sure the display is in the same encdoing as the incoming text"""
storage = six.b('\xf9\xec\xe5\xed') # Hebrew word shalom in cp1255
display = six.b('\xed\xe5\xec\xf9')
self.assertEqual(get_display(storage, encoding='cp1255'), display)
def export (self):
"""Export to ical format"""
start_date = datetime.date(2011, 10, 30) # first sunday of semester
cal = Calendar()
cal.add("prodid", "-//NHY//Technion Students' Schedule 1.0//EN")
cal.add("version", 2.0)
for (day_num, activities) in enumerate(self):
for (activity_num, activity) in enumerate(activities):
# date and time
date = start_date + datetime.timedelta(day_num)
time = datetime.time(*self.split_slot(activity.start))
dt_start = datetime.datetime.combine(date, time)
time = datetime.time(*self.split_slot(activity.start + activity.length))
dt_end = datetime.datetime.combine(date, time)
# add event to calendar file
event = Event()
event.add("summary", activity.text)
event.add("dtstart", dt_start)
event.add("dtend", dt_end)
event.add("dtstamp", datetime.datetime.today())
event["uid"] = "NHY-TECHNION-STUDENTS-SCHEDULE-%d%d" % (day_num, activity_num)
cal.add_component(event)
f = open(os.path.expanduser("~/test.ics"), "w")
f.write(get_display(cal.as_string()))
#f.write((cal.as_string()))
f.close()
def text(self, text):
"""The text to be rendered. Use \\\\n to make new lines.
Issues: May be slow, and pyglet has a memory leak when setting text.
For these reasons, this function checks so that it only updates the
text if it has changed. So scripts can safely set the text on every
frame, with no need to check if it has actually altered.
"""
if text == self.text: # only update for a change
return
if text is not None:
text = str(text) # make sure we have unicode object to render
# deal with some international text issues. Only relevant for Python:
# online experiments use web technologies and handle this seamlessly.
style = self.languageStyle.lower() # be flexible with case
if style == 'arabic' and haveArabic:
# reshape Arabic characters from their isolated form so that
# they flow and join correctly to their neighbours:
text = arabic_reshaper.reshape(text)
if style == 'rtl' or style == 'arabic' and haveArabic:
# deal with right-to-left text presentation by applying the
# bidirectional algorithm:
text = bidi_algorithm.get_display(text)
# no action needed for default 'ltr' (left-to-right) option
self.__dict__['text'] = text
if self.useShaders:
self._setTextShaders(text)
else:
self._setTextNoShaders(text)
self._needSetText = False
def bidi(s):
# convert a multiline string for bidi debug printing
from bidi.algorithm import get_display
s = get_display(s)
lines = s.splitlines()
maxlen = max(len(l) for l in lines)
lines = [('%' + str(maxlen) + 's') % l for l in lines]
return '\n'.join(lines)
def reshape(text):
"""Reshapes arabic in order to display characters from right to left
"""
if platform.system() == "Darwin":
return text
else:
reshaped_text = arabic_reshaper.reshape(text)
bidi_text = get_display(reshaped_text)
return bidi_text
def test_override_base_dir(self):
"""Tests overriding the base paragraph direction"""
# normaly the display should be :MOLAHS be since we're overriding the
# base dir the colon should be at the end of the display
storage = u'SHALOM:'
display = u'MOLAHS:'
self.assertEqual(get_display(storage, upper_is_rtl=True, base_dir='L'), display)
def rtl(exp, reshape=True, bidi=True, digits=False):
if reshape:
exp = reshaper.reshape(exp, digits=digits)
if bidi:
exp = get_display(exp)
return exp
def create_images(label,count=100,font_size = 17):
# here we want to create count images of label with
# different filters/rotations etc..
img = Image.new("L", (100, 100),200)
draw=ImageDraw.Draw(img)
font=ImageFont.truetype("/Library/Fonts/Arial Unicode.ttf", font_size)
label_correct = arabic_reshaper.reshape(label)
label_correct = get_display(label_correct)
draw.text((22,22),label_correct,font=font)
img.save('pics/' +label + '.png')
img.close()
def renderImage(word, label):
reshaped = arabic_reshaper.reshape(u"%s" % word)
bidi = get_display(reshaped)
for f in fList:
#print(f)
font = ImageFont.truetype(fFolder+f, fntSize)
width, height = font.getsize(reshaped)
img = Image.new('RGB', (width, height), "white")
d = ImageDraw.Draw(img)
d.text((0, 0), bidi, fill="black", font=font)
img.save(iFolder+"%s_%s_%s.png" % (label, str(fntSize), f[:-4]))
def process_single_image(word, label, size, fontsize = 50, loc = (25,25)):
if not os.path.exists('pics/' + label):
os.makedirs('pics/' + label)
img = Image.new("L", size ,256) # greyscale and white background
draw = ImageDraw.Draw(img)
font = ImageFont.truetype("/Library/Fonts/Arial Unicode.ttf", fontsize)
label_correct = arabic_reshaper.reshape(word)
label_correct = get_display(label_correct)
draw.text(loc,label_correct,font=font)
direc = 'pics/'+label
img.save(direc + '/original.png')
img.close()
# now we've created the base picture - now we can process it a bit more
create_copies(direc, '/original.png')
def get_farsi_text(text):
if reshaper.has_arabic_letters(text):
words = text.split()
reshaped_words = []
for word in words:
if reshaper.has_arabic_letters(word):
reshaped_text = reshaper.reshape(word)
bidi_text = get_display(reshaped_text)
reshaped_words.append(bidi_text)
else:
reshaped_words.append(word)
reshaped_words.reverse()
return ' '.join(reshaped_words)
return text
def extract(ctx, normalization, reorder, output, transcribs):
"""
Extracts image-text pairs from a transcription environment created using
``ketos transcrib``.
"""
st_time = time.time()
try:
os.mkdir(output)
except:
pass
idx = 0
manifest = []
for fp in transcribs:
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] Reading {}'.format(time.time() - st_time, fp.name))
else:
spin('Reading transcription')
doc = html.parse(fp)
im = None
for section in doc.xpath('//section'):
img = section.xpath('.//img')[0].get('src')
fd = BytesIO(base64.b64decode(img.split(',')[1]))
im = Image.open(fd)
if not im:
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] Skipping {} because image not found'.format(time.time() - st_time, fp.name))
break
for line in section.iter('li'):
if line.get('contenteditable') and u''.join(line.itertext()):
l = im.crop([int(x) for x in line.get('data-bbox').split(',')])
l.save('{}/{:06d}.png'.format(output, idx))
manifest.append('{:06d}.png'.format(idx))
text = u''.join(line.itertext())
if normalization:
text = unicodedata.normalize(normalization, text)
with open('{}/{:06d}.gt.txt'.format(output, idx), 'wb') as t:
if reorder:
t.write(get_display(text).encode('utf-8'))
else:
t.write(text.encode('utf-8'))
idx += 1
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] Extracted {} lines'.format(time.time() - st_time, idx))
with open('{}/manifest.txt'.format(output), 'w') as fp:
fp.write('\n'.join(manifest))
if not ctx.meta['verbose']:
click.secho(u'\b\u2713', fg='green', nl=False)
click.echo('\033[?25h\n', nl=False)
def test_surrogate(self):
"""Test for storage and base levels in case of surrogate pairs"""
storage = get_empty_storage()
text = u'HELLO \U0001d7f612'
get_embedding_levels(text, storage, upper_is_rtl=True)
# should return 9, not 10 even in --with-unicode=ucs2
self.assertEqual(len(storage['chars']), 9)
# Is the expected result ? should be EN
_ch = storage['chars'][6]
self.assertEqual(_ch['ch'], u'\U0001d7f6')
self.assertEqual(_ch['type'], 'EN')
display = get_display(text, upper_is_rtl=True)
self.assertEqual(display, u'\U0001d7f612 OLLEH')
def add(self, image, split=lambda x: os.path.splitext(x)[0],
suffix='.gt.txt', normalization=None, reorder=True,
pad=16):
"""
Adds a single image to the training set.
"""
with click.open_file(split(image) + suffix, 'r', encoding='utf-8') as fp:
gt = fp.read()
if normalization:
gt = unicodedata.normalize(normalization, gt)
if reorder:
gt = bd.get_display(gt)
im = Image.open(image)
im = rpred.dewarp(self.lnorm, im)
im = pil2array(im)
im = lstm.prepare_line(im, pad)
self.training_set.append((im, gt))
def draw_text_using_font(text, font_path):
reshaped_text = arabic_reshaper.reshape(text)
bidi_text = get_display(reshaped_text)
image = Image.new("RGBA", (700, 500), (255, 255, 255))
draw = ImageDraw.Draw(image)
font = ImageFont.truetype(font_path, 100)
draw.text((0, 0), bidi_text, (0, 0, 0), font=font)
width, height = draw.textsize(bidi_text, font=font)
image_frames = []
for i in xrange(len(bidi_text)):
line = bidi_text[:i + 1]
symbol = bidi_text[i:i + 1]
w, _ = draw.textsize(line, font=font)
line_frame = [(0, 0), (w, height)]
symbol_width, _ = draw.textsize(symbol, font=font)
symbol_frame = [(w - symbol_width, 0), (symbol_width, height)]
image_frames.append({'symbol': symbol,
'frame' : symbol_frame})
draw.rectangle(line_frame, outline=(0, 0, 0))
return image, image_frames
def _formatText(self, text):
"Generates PDF text output operator(s)"
canv = self._canvas
font = pdfmetrics.getFont(self._fontname)
R = []
# Adding Unicode BiDirection Support
# Refer to http://github.com/barsi/openerp-rtl
# for more info
try:
text = type(text) == type(u"") and text or text.decode("utf8")
reshaped_text = arabic_reshaper.reshape(text)
text = get_display(reshaped_text)
except UnicodeDecodeError, e:
i, j = e.args[2:4]
raise UnicodeDecodeError(
*(
e.args[:4]
+ ("%s\n%s==[%s]==%s" % (e.args[4], text[max(i - 10, 0) : i], text[i:j], text[j : j + 10]),)
)
)
def test_implict_with_upper_is_rtl(self):
"""Implicit tests"""
tests = (
(u"car is THE CAR in arabic", u"car is RAC EHT in arabic"),
(u"CAR IS the car IN ENGLISH", u"HSILGNE NI the car SI RAC"),
(u'he said "IT IS 123, 456, OK"', u'he said "KO ,456 ,123 SI TI"'),
(u'he said "IT IS (123, 456), OK"', u'he said "KO ,(456 ,123) SI TI"'),
(u'he said "IT IS 123,456, OK"', u'he said "KO ,123,456 SI TI"'),
(u'he said "IT IS (123,456), OK"', u'he said "KO ,(123,456) SI TI"'),
(u'HE SAID "it is 123, 456, ok"', u'"it is 123, 456, ok" DIAS EH'),
(u"<H123>shalom</H123>", u"<123H/>shalom<123H>"),
(u"<h123>SAALAM</h123>", u"<h123>MALAAS</h123>"),
(u'HE SAID "it is a car!" AND RAN', u'NAR DNA "!it is a car" DIAS EH'),
(u'HE SAID "it is a car!x" AND RAN', u'NAR DNA "it is a car!x" DIAS EH'),
(u"SOLVE 1*5 1-5 1/5 1+5", u"1+5 1/5 1-5 5*1 EVLOS"),
(u"THE RANGE IS 2.5..5", u"5..2.5 SI EGNAR EHT"),
(u"-2 CELSIUS IS COLD", u"DLOC SI SUISLEC 2-"),
)
for storage, display in tests:
self.assertEqual(get_display(storage, upper_is_rtl=True), display)
def make_farsi_text(x):
# 针对维语的特殊性，确保维语显示顺序正确
# arabic_reshaper包是根据不同位置时的编码来切割维文
reshaped_text = arabic_reshaper.reshape(x)
farsi_text = get_display(reshaped_text)
return farsi_text
def plot_mandates(self,
bader_ofer,
max_bo=None,
day=0,
hebrew=True,
subtitle=''):
"""
Plot the resulting mandates of the parties and their distributions.
This is the bar graph most often seen in poll results.
"""
from bidi import algorithm as bidialg
fe = self.forecast_model
parties = fe.parties
bo_plot = bader_ofer.transpose(1, 2, 0)[day]
if max_bo is None:
max_bo = self.get_least_square_sum_seats(bader_ofer, day)
num_passed_parties = len(np.where(max_bo > 0)[0])
passed_parties = max_bo.argsort()[::-1]
fig, plots = plt.subplots(2,
num_passed_parties,
figsize=(2 * num_passed_parties, 10),
gridspec_kw={'height_ratios': [5, 1]})
fig.set_facecolor('white')
xlim_dists = []
ylim_height = []
max_bo_height = max_bo.max()
for i in range(num_passed_parties):
party = passed_parties[i]
name = bidialg.get_display(
parties[fe.party_ids[party]]['hname']) if hebrew else parties[
fe.party_ids[party]]['name']
plots[0][i].set_title(name, va='bottom', y=-0.08, fontsize='large')
mandates_count = np.unique(bo_plot[party], return_counts=True)
mandates_bar = plots[0][i].bar(
[0], [max_bo[party]])[0] #, tick_label=[name])
plots[0][i].set_xlim(-0.65, 0.65)
plots[0][i].text(mandates_bar.get_x() +
mandates_bar.get_width() / 2.0,
mandates_bar.get_height(),
'%d' % max_bo[party],
ha='center',
va='bottom',
fontsize='x-large')
plots[0][i].set_ylim(top=max_bo_height)
bars = plots[1][i].bar(
mandates_count[0],
100 * mandates_count[1] / len(bo_plot[party]))
xticks = []
xtick_labels = []
max_start = 0
if 0 in mandates_count[0]:
#xticks += [0]
#xtick_labels += [ '' ]
max_start = 1
zero_rect = bars[0]
zero_rect.set_color('red')
plots[1][i].text(
zero_rect.get_x() + zero_rect.get_width() / 2.0,
zero_rect.get_height(),
' %d%%' %
(100 * mandates_count[1][0] / len(bo_plot[party])),
ha='center',
va='bottom')
if len(mandates_count[1]) > max_start:
interval = self.compute_mandates_interval(bo_plot[party])
max_index = max_start + np.argmax(
mandates_count[1][max_start:])
max_rect = bars[max_index]
#plots[1][i].text(max_rect.get_x() + max_rect.get_width()/2.0, max_rect.get_height(), ' %d%%' % (100 * mandates_count[1][max_index] / len(bo_plot[party])), ha='center', va='bottom')
xticks += [mandates_count[0][max_index]]
xtick_labels += ['\n%d - %d' % interval]
plots[1][i].set_xticks(xticks)
plots[1][i].set_xticklabels(xtick_labels)
xlim = plots[1][i].get_xlim()
xlim_dists += [xlim[1] - xlim[0] + 1]
ylim_height += [plots[1][i].get_ylim()[1]]
plots[0][i].grid(False)
plots[0][i].tick_params(axis='both',
which='both',
left=False,
bottom=False,
labelbottom=False,
labelleft=False)
plots[0][i].set_facecolor('white')
for s in plots[0][i].spines.values():
s.set_visible(False)
plots[1][i].grid(False)
plots[1][i].tick_params(axis='y',
which='both',
left=False,
labelleft=False)
plots[1][i].set_facecolor('white')
for s in plots[1][i].spines.values():
s.set_visible(False)
xlim_side = max(xlim_dists) / 2
for i in range(num_passed_parties):
xlim = plots[1][i].get_xlim()
xlim_center = (xlim[0] + xlim[1]) / 2
plots[1][i].set_xlim(xlim_center - xlim_side,
xlim_center + xlim_side)
plots[1][i].set_ylim(top=max(ylim_height))
bo_mean = bo_plot.mean(axis=1)
failed_parties = [i for i in bo_mean.argsort()[::-1] if max_bo[i] == 0]
num_failed_parties = len(failed_parties)
offset = num_passed_parties // 3
failed_plots = []
for failed_index in range(num_failed_parties):
failed_plot = fig.add_subplot(num_failed_parties * 2,
num_passed_parties,
num_passed_parties *
(failed_index + 1) - offset,
ymargin=1)
party = failed_parties[failed_index]
mandates_count = np.unique(bo_plot[party], return_counts=True)
if len(mandates_count[0]) > 1:
max_start = 0
xticks = []
xtick_labels = []
bars = failed_plot.bar(
mandates_count[0],
100 * mandates_count[1] / len(bo_plot[party]))
if 0 in mandates_count[0]:
#xticks += [0]
max_start = 1
zero_rect = bars[0]
zero_rect.set_color('red')
failed_plot.text(
zero_rect.get_x() + zero_rect.get_width() / 2.0,
zero_rect.get_height(),
' %d%%' %
(100 * mandates_count[1][0] / len(bo_plot[party])),
ha='center',
va='bottom')
if len(mandates_count[1]) > max_start:
interval = self.compute_mandates_interval(bo_plot[party])
max_index = max_start + np.argmax(
mandates_count[1][max_start:])
max_rect = bars[max_index]
#failed_plot.text(max_rect.get_x() + max_rect.get_width()/2.0, max_rect.get_height(), ' %d%%' % (100 * mandates_count[1][max_index] / len(bo_plot[party])), ha='center', va='bottom')
xticks += [mandates_count[0][max_index]]
xtick_labels += ['\n%d - %d' % interval]
failed_plot.set_xticks(xticks)
failed_plot.set_xticklabels(xtick_labels)
else:
failed_plot.text(0.8,
0.5,
str(mandates_count[0][0]),
ha='center',
va='bottom')
name = bidialg.get_display(
parties[fe.party_ids[party]]['hname']) if hebrew else parties[
fe.party_ids[party]]['name']
failed_plot.set_ylabel(name,
va='center',
ha='right',
rotation=0,
fontsize='medium')
failed_plot.yaxis.set_label_position("right")
failed_plot.spines["right"].set_position(("axes", 1.25))
failed_plot.grid(False)
failed_plot.tick_params(axis='both',
which='both',
left=False,
bottom=False,
labelbottom=False,
labelleft=False)
failed_plot.set_facecolor('white')
for s in failed_plot.spines.values():
s.set_visible(False)
failed_plots += [failed_plot]
if num_failed_parties > 0:
max_failed_xlim = max([fp.get_xlim()[1] for fp in failed_plots])
for fp in failed_plots:
fp.set_xlim(right=max_failed_xlim)
fp.set_ylim(0, 150)
title = 'חלוקת המנדטים' if hebrew else 'Mandates Allocation'
if subtitle != '':
title += ' - ' + subtitle
if hebrew:
title = bidialg.get_display(title)
fig.text(.5, 1.05, title, ha='center', fontsize='xx-large')
if fe.house_effects_model is not None:
fig.text(.5,
1.,
self.house_effects_model_title(hebrew),
ha='center',
fontsize='small')
fig.figimage(self.create_logo(),
fig.bbox.xmax / 2 + 50,
fig.bbox.ymax - 100,
zorder=1000)
def line_generator(ctx, font, maxlines, encoding, normalization, renormalize,
reorder, font_size, font_weight, language, max_length,
strip, disable_degradation, binarize, mean, sigma, density,
distort, distortion_sigma, legacy, output, text):
"""
Generates artificial text line training data.
"""
lines = set()
if not text:
return
st_time = time.time()
for t in text:
with click.open_file(t, encoding=encoding) as fp:
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] Reading {}'.format(
time.time() - st_time, t))
else:
spin('Reading texts')
lines.update(fp.readlines())
if normalization:
lines = set(
[unicodedata.normalize(normalization, line) for line in lines])
if strip:
lines = set([line.strip() for line in lines])
if max_length:
lines = set([line for line in lines if len(line) < max_length])
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] Read {} lines'.format(time.time() - st_time,
len(lines)))
else:
click.secho(u'\b\u2713', fg='green', nl=False)
click.echo('\033[?25h\n', nl=False)
click.echo('Read {} unique lines'.format(len(lines)))
if maxlines and maxlines < len(lines):
click.echo('Sampling {} lines\t'.format(maxlines), nl=False)
lines = list(lines)
lines = [
lines[idx] for idx in np.random.randint(0, len(lines), maxlines)
]
click.secho(u'\u2713', fg='green')
try:
os.makedirs(output)
except OSError as e:
if e.errno != errno.EEXIST:
raise
lines = [line.strip() for line in lines]
# calculate the alphabet and print it for verification purposes
alphabet = set()
for line in lines:
alphabet.update(line)
chars = []
combining = []
for char in sorted(alphabet):
if unicodedata.combining(char):
combining.append(unicodedata.name(char))
else:
chars.append(char)
click.echo(u'Σ (len: {})'.format(len(alphabet)))
click.echo(u'Symbols: {}'.format(''.join(chars)))
if combining:
click.echo(u'Combining Characters: {}'.format(', '.join(combining)))
lg = linegen.LineGenerator(font, font_size, font_weight, language)
for idx, line in enumerate(lines):
if ctx.meta['verbose'] > 0:
click.echo(u'[{:2.4f}] {}'.format(time.time() - st_time, line))
else:
spin('Writing images')
try:
if renormalize:
im = lg.render_line(unicodedata.normalize(renormalize, line))
else:
im = lg.render_line(line)
except KrakenCairoSurfaceException as e:
if ctx.meta['verbose'] > 0:
click.echo('[{:2.4f}] {}: {} {}'.format(
time.time() - st_time, e.message, e.width, e.height))
else:
click.secho(u'\b\u2717', fg='red')
click.echo('{}: {} {}'.format(e.message, e.width, e.height))
continue
if not disable_degradation and not legacy:
#im = linegen.distort_line(im, abs(np.random.normal(distort)), abs(np.random.normal(distortion_sigma)))
im = linegen.degrade_line(im)
#im = linegen.degrade_line(im, abs(np.random.normal(mean)), abs(np.random.normal(sigma)), abs(np.random.normal(density)))
elif legacy:
im = linegen.ocropy_degrade(im)
if binarize:
try:
im = binarization.nlbin(im)
except KrakenInputException as e:
click.echo('{}'.format(e.message))
continue
im.save('{}/{:06d}.png'.format(output, idx))
with open('{}/{:06d}.gt.txt'.format(output, idx), 'wb') as fp:
if reorder:
fp.write(get_display(line).encode('utf-8'))
else:
fp.write(line.encode('utf-8'))
if ctx.meta['verbose'] == 0:
click.secho(u'\b\u2713', fg='green', nl=False)
click.echo('\033[?25h\n', nl=False)
def plot_party_support_evolution_graphs(self,
samples,
mbo=None,
burn=None,
hebrew=True):
"""
Plot the evolving support of each party over time in both percentage and seats.
"""
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import matplotlib.dates as mdates
from bidi import algorithm as bidialg
def get_dimensions(n):
divisor = int(np.ceil(np.sqrt(n)))
if n % divisor == 0:
return n // divisor, divisor
else:
return 1 + n // divisor, divisor
if burn is None:
burn = -min(len(samples), 1000)
if mbo is None:
mbo = self.compute_trace_bader_ofer(samples)
samples = samples[burn:]
mbo = mbo[burn:]
fe = self.forecast_model
mbo_by_party = np.transpose(mbo, [2, 1, 0])
mandates = 0
mbo_by_day_party = np.transpose(mbo, [1, 2, 0])
party_avg = mbo_by_day_party[0].mean(axis=1).argsort()[::-1]
date_list = [
fe.forecast_day - datetime.timedelta(days=x)
for x in range(0, fe.num_days)
]
dimensions = get_dimensions(fe.num_parties)
fig, plots = plt.subplots(dimensions[1],
dimensions[0],
figsize=(5.5 * dimensions[0],
3.5 * dimensions[1]))
fig.set_facecolor('white')
means = samples.mean(axis=0)
stds = samples.std(axis=0)
for index, party in enumerate(party_avg):
party_config = fe.config['parties'][fe.party_ids[party]]
if 'created' in party_config:
days_to_show = (fe.forecast_day -
strpdate(party_config['created'])).days
else:
days_to_show = fe.num_days
vindex = index // dimensions[0]
hindex = index % dimensions[0]
if hebrew:
hindex = -hindex - 1 # work right to left in hebrew
subplots = plots[vindex]
subplots[hindex].xaxis.set_major_formatter(
mdates.DateFormatter('%d/%m'))
title = bidialg.get_display(
party_config['hname']) if hebrew else party_config['name']
subplots[hindex].set_title(title)
subplot = subplots[hindex].twinx()
num_day_ticks = fe.num_days
date_ticks = [
fe.forecast_day - datetime.timedelta(days=x)
for x in range(0, num_day_ticks, 7)
]
subplots[hindex].set_xticks(date_ticks)
subplots[hindex].set_xticklabels(
subplots[hindex].get_xticklabels(), rotation=45)
subplots[hindex].set_xlim(date_list[-1], date_list[0])
subplot.set_xticks(date_ticks)
subplot.set_xticklabels(subplot.get_xticklabels(), rotation=45)
subplot.set_xlim(date_list[-1], date_list[0])
party_means = means[:, party]
party_std = stds[:, party]
subplots[hindex].fill_between(
date_list[:days_to_show],
100 * party_means[:days_to_show] -
100 * 1.95996 * party_std[:days_to_show],
100 * party_means[:days_to_show] +
100 * 1.95996 * party_std[:days_to_show],
color='#90ee90')
subplots[hindex].plot(date_list[:days_to_show],
100 * party_means[:days_to_show],
color='#32cd32')
if subplots[hindex].get_ylim()[0] < 0:
subplots[hindex].set_ylim(bottom=0)
subplots[hindex].yaxis.set_major_formatter(
ticker.PercentFormatter(decimals=1))
subplots[hindex].tick_params(axis='y', colors='#32cd32')
subplots[hindex].yaxis.label.set_color('#32cd32')
mand_means = np.mean(mbo_by_party[party], axis=1)
mand_std = np.std(mbo_by_party[party], axis=1)
subplot.fill_between(
date_list[:days_to_show],
mand_means[:days_to_show] - 1.95996 * mand_std[:days_to_show],
mand_means[:days_to_show] + 1.95996 * mand_std[:days_to_show],
alpha=0.5,
color='#6495ed')
subplot.plot(date_list[:days_to_show],
mand_means[:days_to_show],
color='#4169e1')
if subplot.get_ylim()[0] < 0:
subplot.set_ylim(bottom=0)
if subplot.get_ylim()[1] < 4:
subplot.set_ylim(top=4)
if int(subplot.get_ylim()[1]) == int(subplot.get_ylim()[0]):
subplot.set_ylim(top=int(subplot.get_ylim()[0]) + 1)
subplot.yaxis.set_major_locator(
ticker.MaxNLocator(integer=True, min_n_ticks=2, prune=None))
subplot.tick_params(axis='y', colors='#4169e1')
subplot.yaxis.label.set_color('#4169e1')
subplots[hindex].yaxis.tick_right()
subplots[hindex].yaxis.set_label_position("right")
subplots[hindex].spines["right"].set_position(("axes", 1.08))
for s in subplots[hindex].spines.values():
s.set_visible(False)
for s in subplot.spines.values():
s.set_visible(False)
if hindex == dimensions[0] - 1:
subplot.set_ylabel("Seats")
subplots[hindex].set_ylabel("% Support")
subplots[hindex].spines["right"].set_position(("axes", 1.13))
elif hindex == -1:
subplot.set_ylabel(bidialg.get_display("מנדטים"))
subplots[hindex].set_ylabel(bidialg.get_display("אחוזי תמיכה"))
subplots[hindex].spines["right"].set_position(("axes", 1.13))
subplots[hindex].grid(False)
sup = [sample[party][0] for sample in samples]
mandates += int(120.0 * sum(sup) / len(samples))
subplots[hindex].xaxis.set_major_formatter(
mdates.DateFormatter('%d/%m'))
subplots[hindex].set_xlim(date_ticks[-1], date_ticks[0])
subplot.set_xlim(date_ticks[-1], date_ticks[0])
for empty_subplot in range(len(party_avg), np.product(dimensions)):
hindex = empty_subplot % dimensions[0]
if hebrew:
hindex = -hindex - 1 # work right to left in hebrew
plots[-1][hindex].axis('off')
plt.subplots_adjust(wspace=0.5, hspace=0.5)
if hebrew:
title = bidialg.get_display('התמיכה במפלגות לאורך זמן')
else:
title = 'Party Support over Time'
fig.text(.5, 1.05, title, ha='center', fontsize='xx-large')
if fe.house_effects_model is not None:
fig.text(.5,
1.,
self.house_effects_model_title(hebrew),
ha='center',
fontsize='small')
fig.figimage(self.create_logo(),
fig.bbox.xmax / 2 + 100,
fig.bbox.ymax - 100,
zorder=1000)
def get_party_name(party):
namevar = 'hname' if hebrew else 'name'
name = fe.parties[party][namevar]
if 'subsets' in config and party in config['subsets']:
name += ' (%s)' % config['subsets'][party][namevar]
return bidialg.get_display(name) if hebrew else name
def convert(text):
new_text = arabic_reshaper.reshape(text)
bidi_text = get_display(new_text)
return bidi_text
def recognize(self, img_cv_grey, horizontal_list=None, free_list=None,\
decoder = 'greedy', beamWidth= 5, batch_size = 1,\
workers = 0, allowlist = None, blocklist = None, detail = 1,\
rotation_info = None,\
paragraph = False,\
contrast_ths = 0.1,adjust_contrast = 0.5, filter_ths = 0.003,\
reformat=True):
if reformat:
img, img_cv_grey = reformat_input(img_cv_grey)
if (horizontal_list == None) and (free_list == None):
y_max, x_max = img_cv_grey.shape
ratio = x_max / y_max
max_width = int(imgH * ratio)
crop_img = cv2.resize(img_cv_grey, (max_width, imgH),
interpolation=Image.ANTIALIAS)
image_list = [([[0, 0], [x_max, 0], [x_max, y_max],
[0, y_max]], crop_img)]
else:
image_list, max_width = get_image_list(horizontal_list,
free_list,
img_cv_grey,
model_height=imgH)
if allowlist:
ignore_char = ''.join(set(self.character) - set(allowlist))
elif blocklist:
ignore_char = ''.join(set(blocklist))
else:
ignore_char = ''.join(set(self.character) - set(self.lang_char))
image_len = len(image_list)
if rotation_info and image_list:
image_list = make_rotated_img_list(rotation_info, image_list)
if self.model_lang in [
'chinese_tra', 'chinese_sim', 'japanese', 'korean'
]:
decoder = 'greedy'
result = get_text(self.character, imgH, int(max_width), self.recognizer, self.converter, image_list,\
ignore_char, decoder, beamWidth, batch_size, contrast_ths, adjust_contrast, filter_ths,\
workers, self.device)
if self.model_lang == 'arabic':
direction_mode = 'rtl'
result = [list(item) for item in result]
for item in result:
item[1] = get_display(item[1])
else:
direction_mode = 'ltr'
if paragraph:
result = get_paragraph(result, mode=direction_mode)
if rotation_info and image_list:
result = set_result_with_confidence(result, image_len)
if detail == 0:
return [item[1] for item in result]
else:
return result
def readtext(self, image, decoder = 'greedy', beamWidth= 5, batch_size = 1,\
workers = 0, allowlist = None, blocklist = None, detail = 1,\
paragraph = False,\
contrast_ths = 0.1,adjust_contrast = 0.5, filter_ths = 0.003,\
text_threshold = 0.7, low_text = 0.4, link_threshold = 0.4,\
canvas_size = 2560, mag_ratio = 1.,\
slope_ths = 0.1, ycenter_ths = 0.5, height_ths = 0.5,\
width_ths = 0.5, add_margin = 0.1):
'''
Parameters:
file: file path or numpy-array or a byte stream object
'''
if type(image) == str:
if image.startswith('http://') or image.startswith('https://'):
tmp, _ = urlretrieve(image)
img_cv_grey = cv2.imread(tmp, cv2.IMREAD_GRAYSCALE)
os.remove(tmp)
else:
img_cv_grey = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
image = os.path.expanduser(image)
img = loadImage(image) # can accept URL
elif type(image) == bytes:
nparr = np.frombuffer(image, np.uint8)
img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_cv_grey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
elif type(image) == np.ndarray:
if len(image.shape) == 2: # grayscale
img_cv_grey = image
img = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
elif len(image.shape) == 3: # BGRscale
img = image
img_cv_grey = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
text_box = get_textbox(self.detector, img, canvas_size, mag_ratio, text_threshold,\
link_threshold, low_text, False, self.device)
horizontal_list, free_list = group_text_box(text_box, slope_ths,
ycenter_ths, height_ths,
width_ths, add_margin)
# should add filter to screen small box out
image_list, max_width = get_image_list(horizontal_list,
free_list,
img_cv_grey,
model_height=imgH)
if allowlist:
ignore_char = ''.join(set(self.character) - set(allowlist))
elif blocklist:
ignore_char = ''.join(set(blocklist))
else:
ignore_char = ''.join(set(self.character) - set(self.lang_char))
if self.model_lang in [
'chinese_tra', 'chinese_sim', 'japanese', 'korean'
]:
decoder = 'greedy'
result = get_text(self.character, imgH, int(max_width), self.recognizer, self.converter, image_list,\
ignore_char, decoder, beamWidth, batch_size, contrast_ths, adjust_contrast, filter_ths,\
workers, self.device)
if self.model_lang == 'arabic':
result = [list(item) for item in result]
for item in result:
item[1] = get_display(item[1])
if paragraph:
result = get_paragraph(result)
if detail == 0:
return [item[1] for item in result]
else:
return result
doc = nlp(text_to_be_reshaped)
text = []
for each in doc:
if str(each) not in str(stop_words.STOP_WORDS):
#(each)
text.append(str(each))
reshaped_text = ""
for each in text:
reshaped_text = reshaped_text + " " + each
reshaped_text = reshaper.reshape(reshaped_text)
from bidi.algorithm import get_display
bidi_text = get_display(reshaped_text)
fontdir = "F:\\FinalSemester\\FYP_2_Deploymenty\\New folder\\project_updated_6thFeb\\Urdu_fonts\\"
import os
fonts = os.listdir(fontdir)
x = 1
for font in fonts:
print(x)
if font.endswith("ttf"):
print("Font", font)
plt.title = "Word Cloud"
plt.figure(figsize=(20, 15), dpi=400)
wordcloud = WordCloud(
def __repr__(self):
return algorithm.get_display(self.username)#self.username if all(ord(c) < 128 for c in self.username) else self.username[::-1]
from bidi.algorithm import get_display
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
for i in range(86):
packet = io.BytesIO()
# create a new PDF with Reportlab
can = canvas.Canvas(packet, pagesize=letter)
text = raw_input().decode('utf-8')
#reportlab.rl_config.TTFSearchPath.append(str(settings.BASE_DIR))
fontDir = '/home/mrgafs/.fonts/DTHULUTH-II-1.ttf'
pdfmetrics.registerFont(TTFont('font',fontDir))
can.setFont("font", 55)
can.setFillColor(HexColor(0x862DAC))
ar = arabic_reshaper.reshape(text)
ar = get_display(ar)
can.drawString(240, 356, ar.encode('utf-8'))
can.save()
#move to the beginning of the StringIO buffer
packet.seek(0)
new_pdf = PdfFileReader(packet)
# read your existing PDF
certDir= "/home/mrgafs/Desktop/Certificates/automate-the-boring-stuff-with-python-2015-.pdf"
existing_pdf = PdfFileReader(open(certDir , "rb"))
output = PdfFileWriter()
# add the "watermark" (which is the new pdf) on the existing page
page = existing_pdf.getPage(0)
page.mergePage(new_pdf.getPage(0))
output.addPage(page)
# finally, write "output" to a real file
outputStream = open(text+".pdf", "wb")
def line_generator(ctx, font, maxlines, encoding, normalization, renormalize,
reorder, font_size, font_weight, language, max_length, strip,
disable_degradation, alpha, beta, distort, distortion_sigma,
legacy, output, text):
"""
Generates artificial text line training data.
"""
import errno
import numpy as np
from kraken import linegen
from kraken.lib.util import make_printable
lines: Set[str] = set()
if not text:
return
with log.progressbar(text, label='Reading texts') as bar:
for t in text:
with click.open_file(t, encoding=encoding) as fp:
logger.info('Reading {}'.format(t))
for l in fp:
lines.add(l.rstrip('\r\n'))
if normalization:
lines = set([unicodedata.normalize(normalization, line) for line in lines])
if strip:
lines = set([line.strip() for line in lines])
if max_length:
lines = set([line for line in lines if len(line) < max_length])
logger.info('Read {} lines'.format(len(lines)))
message('Read {} unique lines'.format(len(lines)))
if maxlines and maxlines < len(lines):
message('Sampling {} lines\t'.format(maxlines), nl=False)
llist = list(lines)
lines = set(llist[idx] for idx in np.random.randint(0, len(llist), maxlines))
message('\u2713', fg='green')
try:
os.makedirs(output)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# calculate the alphabet and print it for verification purposes
alphabet: Set[str] = set()
for line in lines:
alphabet.update(line)
chars = []
combining = []
for char in sorted(alphabet):
k = make_printable(char)
if k != char:
combining.append(k)
else:
chars.append(k)
message('Σ (len: {})'.format(len(alphabet)))
message('Symbols: {}'.format(''.join(chars)))
if combining:
message('Combining Characters: {}'.format(', '.join(combining)))
lg = linegen.LineGenerator(font, font_size, font_weight, language)
with log.progressbar(lines, label='Writing images') as bar:
for idx, line in enumerate(bar):
logger.info(line)
try:
if renormalize:
im = lg.render_line(unicodedata.normalize(renormalize, line))
else:
im = lg.render_line(line)
except KrakenCairoSurfaceException as e:
logger.info('{}: {} {}'.format(e.message, e.width, e.height))
continue
if not disable_degradation and not legacy:
im = linegen.degrade_line(im, alpha=alpha, beta=beta)
im = linegen.distort_line(im, abs(np.random.normal(distort)), abs(np.random.normal(distortion_sigma)))
elif legacy:
im = linegen.ocropy_degrade(im)
im.save('{}/{:06d}.png'.format(output, idx))
with open('{}/{:06d}.gt.txt'.format(output, idx), 'wb') as fp:
if reorder:
fp.write(get_display(line).encode('utf-8'))
else:
fp.write(line.encode('utf-8'))
def OnConvert(self, evt):
self.tc2.Clear()
reshaped_text = arabic_reshaper.reshape(self.tc1.GetValue())
bidi_text = get_display(reshaped_text)
self.tc2.WriteText(bidi_text)
def line_generator(ctx, font, maxlines, encoding, normalization, renormalize,
reorder, font_size, font_weight, language, max_length,
strip, disable_degradation, alpha, beta, distort,
distortion_sigma, legacy, output, text):
"""
Generates artificial text line training data.
"""
import errno
import numpy as np
from kraken import linegen
from kraken.lib.util import make_printable
lines: Set[str] = set()
if not text:
return
with log.progressbar(text, label='Reading texts') as bar:
for t in text:
with click.open_file(t, encoding=encoding) as fp:
logger.info('Reading {}'.format(t))
for l in fp:
lines.add(l.rstrip('\r\n'))
if normalization:
lines = set(
[unicodedata.normalize(normalization, line) for line in lines])
if strip:
lines = set([line.strip() for line in lines])
if max_length:
lines = set([line for line in lines if len(line) < max_length])
logger.info('Read {} lines'.format(len(lines)))
message('Read {} unique lines'.format(len(lines)))
if maxlines and maxlines < len(lines):
message('Sampling {} lines\t'.format(maxlines), nl=False)
llist = list(lines)
lines = set(llist[idx]
for idx in np.random.randint(0, len(llist), maxlines))
message('\u2713', fg='green')
try:
os.makedirs(output)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# calculate the alphabet and print it for verification purposes
alphabet: Set[str] = set()
for line in lines:
alphabet.update(line)
chars = []
combining = []
for char in sorted(alphabet):
k = make_printable(char)
if k != char:
combining.append(k)
else:
chars.append(k)
message('Σ (len: {})'.format(len(alphabet)))
message('Symbols: {}'.format(''.join(chars)))
if combining:
message('Combining Characters: {}'.format(', '.join(combining)))
lg = linegen.LineGenerator(font, font_size, font_weight, language)
with log.progressbar(lines, label='Writing images') as bar:
for idx, line in enumerate(bar):
logger.info(line)
try:
if renormalize:
im = lg.render_line(
unicodedata.normalize(renormalize, line))
else:
im = lg.render_line(line)
except KrakenCairoSurfaceException as e:
logger.info('{}: {} {}'.format(e.message, e.width, e.height))
continue
if not disable_degradation and not legacy:
im = linegen.degrade_line(im, alpha=alpha, beta=beta)
im = linegen.distort_line(
im, abs(np.random.normal(distort)),
abs(np.random.normal(distortion_sigma)))
elif legacy:
im = linegen.ocropy_degrade(im)
im.save('{}/{:06d}.png'.format(output, idx))
with open('{}/{:06d}.gt.txt'.format(output, idx), 'wb') as fp:
if reorder:
fp.write(get_display(line).encode('utf-8'))
else:
fp.write(line.encode('utf-8'))
datf = pd.DataFrame({
'root_counts': root_counts,
'root_sura_counts': root_sura_counts,
'root_sura_freq': root_sura_freq
})
print(datf)
fig, ax = plt.subplots()
plt.scatter(datf['root_sura_counts'],
datf['root_sura_freq'],
s=datf['root_counts'],
alpha=0.2) # hue='root_counts'
for n, xs, ys in zip(datf.index, datf['root_sura_counts'],
datf['root_sura_freq']):
ax.text(xs,
ys,
bidialg.get_display(arabic_reshaper.reshape(n)),
alpha=0.6,
size=10) ##bidialg.get_display(arabic_reshaper.reshape(n))
plt.xlabel('Total number of Suras in which root appears')
plt.ylabel(
'Total root appearances divided by Total number of Suras in which root appears'
)
plt.show()
plt.close()
if True:
ax = sns.kdeplot(datf['root_sura_counts'],
datf['root_sura_freq'],
shade=True)
plt.show()
plt.close()
def recognize(self, img_cv_grey, horizontal_list=None, free_list=None,\
decoder = 'greedy', beamWidth= 5, batch_size = 1,\
workers = 0, allowlist = None, blocklist = None, detail = 1,\
rotation_info = None,\
paragraph = False,\
contrast_ths = 0.1,adjust_contrast = 0.5, filter_ths = 0.003,\
reformat=True):
if reformat:
img, img_cv_grey = reformat_input(img_cv_grey)
if allowlist:
ignore_char = ''.join(set(self.character) - set(allowlist))
elif blocklist:
ignore_char = ''.join(set(blocklist))
else:
ignore_char = ''.join(set(self.character) - set(self.lang_char))
if self.model_lang in ['chinese_tra', 'chinese_sim']:
decoder = 'greedy'
if (horizontal_list == None) and (free_list == None):
y_max, x_max = img_cv_grey.shape
horizontal_list = [[0, x_max, 0, y_max]]
free_list = []
# without gpu/parallelization, it is faster to process image one by one
if ((batch_size == 1) or (self.device == 'cpu')) and not rotation_info:
result = []
for bbox in horizontal_list:
h_list = [bbox]
f_list = []
image_list, max_width = get_image_list(h_list,
f_list,
img_cv_grey,
model_height=imgH)
result0 = get_text(self.character, imgH, int(max_width), self.recognizer, self.converter, image_list,\
ignore_char, decoder, beamWidth, batch_size, contrast_ths, adjust_contrast, filter_ths,\
workers, self.device)
result += result0
for bbox in free_list:
h_list = []
f_list = [bbox]
image_list, max_width = get_image_list(h_list,
f_list,
img_cv_grey,
model_height=imgH)
result0 = get_text(self.character, imgH, int(max_width), self.recognizer, self.converter, image_list,\
ignore_char, decoder, beamWidth, batch_size, contrast_ths, adjust_contrast, filter_ths,\
workers, self.device)
result += result0
# default mode will try to process multiple boxes at the same time
else:
image_list, max_width = get_image_list(horizontal_list,
free_list,
img_cv_grey,
model_height=imgH)
image_len = len(image_list)
if rotation_info and image_list:
image_list = make_rotated_img_list(rotation_info, image_list)
max_width = max(max_width, imgH)
result = get_text(self.character, imgH, int(max_width), self.recognizer, self.converter, image_list,\
ignore_char, decoder, beamWidth, batch_size, contrast_ths, adjust_contrast, filter_ths,\
workers, self.device)
if rotation_info and (horizontal_list + free_list):
result = set_result_with_confidence(result, image_len)
if self.model_lang == 'arabic':
direction_mode = 'rtl'
result = [list(item) for item in result]
for item in result:
item[1] = get_display(item[1])
else:
direction_mode = 'ltr'
if paragraph:
result = get_paragraph(result, mode=direction_mode)
if detail == 0:
return [item[1] for item in result]
else:
return result
def main(data_of_time_table, part=None):
mpl.interactive(False)
days = []
starts = []
ends = []
coms = []
darss = []
colors = [
'pink', 'lightgreen', 'lightblue', 'crimson', 'salmon', 'yellow',
'silver', 'yellowgreen', 'y', 'tan', 'orchid', 'c', 'aqua', 'deeppink'
]
for line in data_of_time_table:
data = line.split(' | ')
days.append(get_display(persian_reshaper.reshape(data[0])))
starts.append(
float(data[1].split(':')[0]) + float(data[1].split(':')[1]) / 60)
ends.append(
float(data[2].split(':')[0]) + float(data[2].split(':')[1]) / 60)
if len(data) == 6:
coms.append(get_display(persian_reshaper.reshape(data[3])))
darss.append(get_display(persian_reshaper.reshape(data[4])))
days2 = set(days)
prop = fm.FontProperties(
fname=os.path.dirname(os.path.abspath(__file__)) +
'/fonts/XP_ZibaBd.ttf') # os.path.abspath('app/fonts/XP_ZibaBd.ttf'))
dd = dict((ds(x), x) for x in days2)
sorted_days = []
for key in sorted(dd):
sorted_days.append(dd[key])
darss = list(set(darss))
nl = dict((darss[x], x) for x in range(len(darss)))
mn = min(starts)
mx = max(ends)
fig = plt.figure(figsize=(16, 9))
gc.collect()
for i, line in zip(range(len(days)), data_of_time_table):
data = line.split(' | ')
this_start = float(
data[1].split(':')[0]) + float(data[1].split(':')[1]) / 60
this_end = float(
data[2].split(':')[0]) + float(data[2].split(':')[1]) / 60
temp3 = []
if len(data[3]) > 200 * (this_end - this_start) / (mx - mn): # 300
for iji in range(
int(
len(data[3]) / (213 * (this_end - this_start) /
(mx - mn))) + 1): # 32
temp3.append(
data[3][iji *
int(213 * (this_end - this_start) /
(mx - mn)):int(213 * (this_end - this_start) /
(mx - mn)) * (iji + 1)]) # 32
data[3] = '\n'.join(temp3)
temp5 = []
if len(data[5]) > (213 * (this_end - this_start) / (mx - mn)): # 32
for ijl in range(
int(
len(data[5]) / (226 * (this_end - this_start) /
(mx - mn))) + 1): # 34
temp5.append(
data[5][ijl *
int(226 * (this_end - this_start) /
(mx - mn)):int(226 * (this_end - this_start) /
(mx - mn)) * (ijl + 1)]) # 34
data[5] = '\n'.join(temp5)
if part is None:
c = colors[nl[get_display(persian_reshaper.reshape(data[4]))]]
elif days[i] == get_display(
persian_reshaper.reshape(part.split(' | ')[0])) and float(
part.split(' | ')[1].split(':')[0]) + float(
part.split(' | ')[1].split(':')[1]) / 60 == this_start:
c = 'pink'
else:
c = 'whitesmoke'
plt.fill_between([starts[i] + 0.05, ends[i] - 0.05], [
sorted_days.index(days[i]) + 0.53,
sorted_days.index(days[i]) + 0.53
], [
sorted_days.index(days[i]) + 1.3,
sorted_days.index(days[i]) + 1.43
],
color=c,
edgecolor='red',
linewidth=5)
temp4 = []
if len(data[4]) > (150 * (this_end - this_start) / (mx - mn)): # 24
for ijk in range(
int(
len(data[4]) / (180 * (this_end - this_start) /
(mx - mn))) + 1): # 27
temp4.append(
data[4][ijk *
int(180 * (this_end - this_start) /
(mx - mn)):int(180 * (this_end - this_start) /
(mx - mn)) * (ijk + 1)]) # 27
data[4] = '\n'.join(temp4)
plt.text(starts[i] + 0.25,
sorted_days.index(days[i]) + 0.55,
'{0}:{1:0>2}'.format(int(data[1].split(':')[0]),
int(data[1].split(':')[1])),
va='top',
fontsize=7)
plt.text(ends[i] - 0.05,
sorted_days.index(days[i]) + 0.55,
'{0}:{1:0>2}'.format(int(data[2].split(':')[0]),
int(data[2].split(':')[1])),
va='top',
fontsize=7)
plt.text((starts[i] + ends[i]) * 0.5,
(sorted_days.index(days[i]) + 0.5 +
sorted_days.index(days[i]) + 1.4) * 0.5 - 0.2,
get_display(persian_reshaper.reshape(data[4])),
ha='center',
va='center',
fontproperties=prop,
fontsize=13 - 1.5 * len(temp4))
plt.text(
(starts[i] + ends[i]) * 0.5,
(sorted_days.index(days[i]) + sorted_days.index(days[i]) + 2.05) *
0.5,
get_display(persian_reshaper.reshape(data[3])),
ha='center',
va='center',
fontsize=9 - len(temp3))
plt.text(
(starts[i] + ends[i]) * 0.5,
(sorted_days.index(days[i]) + sorted_days.index(days[i]) + 2) * 0.5
+ 0.23,
get_display(persian_reshaper.reshape(data[5])),
ha='center',
va='center',
fontsize=8 - len(temp5))
gc.collect()
"""
examsb = []
for line in user_data['exams']:
pattern = r'^.*(?P<y>\d{4})\/(?P<m>\d{2})\/(?P<d>\d{2}).*از ((?P<h>\d{2})\:\d{2}) تا.*$'
examsb.append(re.search(pattern, line))
exams = dict(
(int(k.group('h')) / 24 + int(k.group('d')) + (int(k.group('m')) - 1) * 31 + (int(k.group('y')) - 1) * 31 * 12,
list()) for k in examsb
if k is not None)
[exams[int(k.group('h')) / 24 + int(k.group('d')) + (int(k.group('m')) - 1) * 31 + (
int(k.group('y')) - 1) * 31 * 12].append(k.group()) for k
in examsb if k is not None]
mtermsb = []
for line in user_data['midterm']:
pattern = r'^.*(?P<y>\d{4})\/(?P<m>\d{1,2})\/(?P<d>\d{1,2}).*$'
mtermsb.append(re.search(pattern, line))
mterms = dict(
(int(k.group('d')) + (int(k.group('m')) - 1) * 31 + (int(k.group('y')) - 1) * 31 * 12,
list()) for k in mtermsb
if k is not None)
[mterms[int(k.group('d')) + (int(k.group('m')) - 1) * 31 + (
int(k.group('y')) - 1) * 31 * 12].append(k.group()) for k
in mtermsb if k is not None]
jj = 0
exx = mx
plt.text(exx + 0.5, (1 + 5.5 * len(sorted_days)) * 0.2, get_display(persian_reshaper.reshape(':پایانترم')),
fontsize=13)
for key in sorted(exams):
for j in exams[key]:
if jj == 8:
exx -= 4
jj = 0
plt.text(exx, (jj + 1.5 + 5.5 * len(sorted_days)) * 0.2, get_display(persian_reshaper.reshape(j)),
fontsize=11)
jj += 1
if from_scrp:
mterms = []
if not from_scrp:
if 'midterm' in user_data:
plt.text(mn - 0.05, (1 + 5.5 * len(sorted_days)) * 0.2, get_display(persian_reshaper.reshape('میانترم:')),
fontsize=11)
jj2 = 0
exx = mn + 3
for key in sorted(mterms):
for j in mterms[key]:
if jj2 == 8:
exx += 4
jj2 = 0
plt.text(exx, (jj2 + 1.5 + 5.5 * len(sorted_days)) * 0.2, get_display(persian_reshaper.reshape(j)),
fontsize=11)
jj2 += 1
"""
ax = fig.add_subplot(111)
ax.set_ylim(len(days2) + 2 / 4, 0.4)
ax.set_xlim(mx, mn)
ax.set_yticks(range(1, len(days2) + 1))
ax.set_yticklabels(sorted_days)
ax.xaxis.grid()
ax2 = ax.twiny().twinx()
ax2.set_ylim(ax.get_ylim())
ax2.set_xlim(ax.get_xlim())
ax2.set_yticks(ax.get_yticks())
ax2.set_yticklabels(sorted_days, fontsize=13)
plt.subplots_adjust(left=0.06,
bottom=0.26,
right=0.95,
top=0.96,
wspace=0.2,
hspace=0.2)
buf = io.BytesIO()
plt.savefig(buf, format='svg', bbox_inches='tight')
s = buf.getvalue()
buf.close()
plt.cla()
plt.clf()
ax.cla()
ax2.cla()
gc.collect()
plt.close('all')
return s
def printit_ar(pname, ti, ofc, tnu, tas, cticket):
def fsizeit(text, t, f):
ltxt = "A" * len(t)
return f.getsize(t)
def center(text, t, f):
fs1, fs2 = fsizeit(text, t, f)
return ((text.size[0] - fs1) / 2, (text.size[1] - fs2) / 2)
if name == 'nt':
fpath = r_path('static\\gfonts\\arial.ttf')
else:
fpath = r_path('static/gfonts/arial.ttf')
fonts = [
ImageFont.truetype(fpath, 50),
ImageFont.truetype(fpath, 30),
ImageFont.truetype(fpath, 25)
]
logo = 'FQM ' + version[:4]
title = u'نظام إدارة الحشود الحر'
title = arabic_reshaper.reshape(title)
title = get_display(title)
link = 'http://fqms.github.io'
border = "#" * 20
ticket = str(ti)
officet = u'المكتب : ' + ofc
office = arabic_reshaper.reshape(officet)
office = get_display(office)
try:
taskt = u'المهمة : ' + tas
except:
taskt = tas
task = arabic_reshaper.reshape(taskt)
task = get_display(task)
datet = u'الوقت : '
datet += str(datetime.utcnow())[:-7]
date = arabic_reshaper.reshape(datet)
date = get_display(date)
aheadt = u'تذاكر قبلك : '
aheadt += str(tnu)
ahead = arabic_reshaper.reshape(aheadt)
ahead = get_display(ahead)
cutit = u'التذكرة الحالية : '
cutit += str(cticket)
cuti = arabic_reshaper.reshape(cutit)
cuti = get_display(cuti)
w = 400
bt_1 = Image.new('RGB', (w, 60), "white")
bt_2 = Image.new('RGB', (w, 60), "white")
bt_3 = Image.new('RGB', (w, 60), "white")
st_1 = Image.new('RGB', (w, 50), "white")
st_2 = Image.new('RGB', (w, 50), "white")
st_3 = Image.new('RGB', (w, 50), "white")
st_4 = Image.new('RGB', (w, 50), "white")
st_5 = Image.new('RGB', (w, 50), "white")
st_6 = Image.new('RGB', (w, 50), "white")
st_7 = Image.new('RGB', (w, 50), "white")
st_8 = Image.new('RGB', (w, 50), "white")
tt = 50 * 8
tt += 60 * 3
mt = Image.new('RGB', (w, tt), "white")
bd_1 = ImageDraw.Draw(bt_1)
bd_2 = ImageDraw.Draw(bt_2)
bd_3 = ImageDraw.Draw(bt_3)
sd_1 = ImageDraw.Draw(st_1)
sd_2 = ImageDraw.Draw(st_2)
sd_3 = ImageDraw.Draw(st_3)
sd_4 = ImageDraw.Draw(st_4)
sd_5 = ImageDraw.Draw(st_5)
sd_6 = ImageDraw.Draw(st_6)
sd_7 = ImageDraw.Draw(st_7)
sd_8 = ImageDraw.Draw(st_8)
md = ImageDraw.Draw(mt)
b = "black"
bd_1.text(center(bt_1, logo, fonts[0]), logo, font=fonts[0], fill=b)
bd_2.text(center(bt_2, title, fonts[1]), title, font=fonts[1], fill=b)
bd_3.text(center(bt_3, ticket, fonts[0]), ticket, font=fonts[0], fill=b)
sd_1.text(center(st_1, link, fonts[2]), link, font=fonts[2], fill=b)
sd_2.text(center(st_2, border, fonts[2]), border, font=fonts[2], fill=b)
sd_3.text(center(st_3, border, fonts[2]), border, font=fonts[2], fill=b)
sd_4.text(center(st_4, office, fonts[2]), office, font=fonts[2], fill=b)
sd_5.text(center(st_5, task, fonts[2]), task, font=fonts[2], fill=b)
sd_6.text(center(st_6, date, fonts[2]), date, font=fonts[2], fill=b)
sd_7.text(center(st_7, ahead, fonts[2]), ahead, font=fonts[2], fill=b)
sd_8.text(center(st_8, cuti, fonts[2]), cuti, font=fonts[2], fill=b)
tts = 0
mt.paste(bt_1, (0, 0))
tts += bt_1.size[1]
mt.paste(bt_2, (0, tts))
tts += bt_2.size[1]
mt.paste(st_1, (0, tts))
tts += st_1.size[1]
mt.paste(st_2, (0, tts))
tts += st_2.size[1]
mt.paste(bt_3, (0, tts))
tts += bt_3.size[1]
mt.paste(st_3, (0, tts))
tts += st_3.size[1]
mt.paste(st_4, (0, tts))
tts += st_4.size[1]
mt.paste(st_8, (0, tts))
tts += st_8.size[1]
mt.paste(st_7, (0, tts))
tts += st_7.size[1]
mt.paste(st_5, (0, tts))
tts += st_5.size[1]
mt.paste(st_6, (0, tts))
iname = 'dummy.jpg'
finame = path.join(getcwd(), iname)
mt.save(iname, format="JPEG")
pname.image(iname, fragment_height=tt, high_density_vertical=True)
pname.cut()
pname.close()
if path.isfile(finame):
remove(finame)
result = [word for word in text2 if word not in arabic_words]
final_result = ' '.join(result)
return final_result
with open('LamaAlherbish_tweets.json') as f:
print('hh')
data_file = json.load(f)
# print(data_file)
for i in data_file:
cleaned_text = clean_Text(str=i['text'])
tweets_data.append(cleaned_text.split())
tweets_data2 = list(chain.from_iterable(tweets_data))
twenty_most_commom_words_with_frequency = Counter(tweets_data2).most_common(20)
for word in twenty_most_commom_words_with_frequency:
print(word[0])
twenty_most_commom_words.append(word[0])
# # convert list to string and generate
unique_string = (" ").join(twenty_most_commom_words)
reshaped_texts = arabic_reshaper.reshape(unique_string)
reshaped_texts = get_display(reshaped_texts)
wordcloud = WordCloud(font_path='Fonts/Supplemental/Damascus.ttc', width=700, height=300, background_color="white").generate(reshaped_texts)
plt.axis('off')
plt.imshow(wordcloud, interpolation='bilinear')
plt.savefig('LamaAlherbish_wordCloud.png')
def text_to_bidi(text):
text = normalize_spaces(text)
reshaped_text = arabicreshaper.reshape(text)
bidi_text = get_display(reshaped_text)
return bidi_text
positive = [x + y for x, y in zip(happy, surprised)]
#---------------------------------------------------
# draw plot : positive,neutral and negative
plt.stackplot(ph_,
positive,
neutral,
negative,
colors=pal,
alpha=0.4)
#plt.xticks(rotation=45)
#---------------------------------------------------
# Prevent x-tick labels from overlapping
ax.xaxis.set_major_locator(ticker.AutoLocator())
positve_label = get_display(arabic_reshaper.reshape('إيجابي'))
neutral_label = get_display(arabic_reshaper.reshape('محايد'))
negative_label = get_display(arabic_reshaper.reshape('سلبي'))
plt.plot([], [],
label=positve_label,
color=pal[0],
marker='s',
linestyle='None',
markersize=10)
plt.plot([], [],
label=neutral_label,
color=pal[1],
marker='s',
linestyle='None',
markersize=10)
plt.plot([], [],
def print_ticket_cli_ar(pname, ti, ofc, tnu, tas, cticket, host='localhost', **kwargs):
def fsizeit(text, t, f):
return f.getsize(t)
def center(text, t, f):
fs1, fs2 = fsizeit(text, t, f)
return ((text.size[0] - fs1) / 2, (text.size[1] - fs2) / 2)
fpath = absolute_path('static/gfonts/arial.ttf')
fonts = [ImageFont.truetype(fpath, 50),
ImageFont.truetype(fpath, 30),
ImageFont.truetype(fpath, 25)]
logo = 'FQM ' + VERSION[:4]
title = u'نظام إدارة الحشود الحر'
title = arabic_reshaper.reshape(title)
title = get_display(title)
link = 'http://fqms.github.io'
border = "#" * 20
ticket = str(ti)
officet = u'المكتب : ' + ofc
office = arabic_reshaper.reshape(officet)
office = get_display(office)
try:
taskt = u'المهمة : ' + tas
except Exception:
taskt = tas
task = arabic_reshaper.reshape(taskt)
task = get_display(task)
datet = u'الوقت : '
datet += str(datetime.now())[:-7]
date = arabic_reshaper.reshape(datet)
date = get_display(date)
aheadt = u'تذاكر قبلك : '
aheadt += str(tnu)
ahead = arabic_reshaper.reshape(aheadt)
ahead = get_display(ahead)
cutit = u'التذكرة الحالية : '
cutit += str(cticket)
cuti = arabic_reshaper.reshape(cutit)
cuti = get_display(cuti)
w = 400
bt_1 = Image.new('RGB', (w, 60), "white")
bt_2 = Image.new('RGB', (w, 60), "white")
bt_3 = Image.new('RGB', (w, 60), "white")
st_1 = Image.new('RGB', (w, 50), "white")
st_2 = Image.new('RGB', (w, 50), "white")
st_3 = Image.new('RGB', (w, 50), "white")
st_4 = Image.new('RGB', (w, 50), "white")
st_5 = Image.new('RGB', (w, 50), "white")
st_6 = Image.new('RGB', (w, 50), "white")
st_7 = Image.new('RGB', (w, 50), "white")
st_8 = Image.new('RGB', (w, 50), "white")
tt = 50 * 8
tt += 60 * 3
mt = Image.new('RGB', (w, tt), "white")
bd_1 = ImageDraw.Draw(bt_1)
bd_2 = ImageDraw.Draw(bt_2)
bd_3 = ImageDraw.Draw(bt_3)
sd_1 = ImageDraw.Draw(st_1)
sd_2 = ImageDraw.Draw(st_2)
sd_3 = ImageDraw.Draw(st_3)
sd_4 = ImageDraw.Draw(st_4)
sd_5 = ImageDraw.Draw(st_5)
sd_6 = ImageDraw.Draw(st_6)
sd_7 = ImageDraw.Draw(st_7)
sd_8 = ImageDraw.Draw(st_8)
md = ImageDraw.Draw(mt)
b = "black"
bd_1.text(center(bt_1, logo, fonts[0]), logo,
font=fonts[0], fill=b)
bd_2.text(center(bt_2, title, fonts[1]), title,
font=fonts[1], fill=b)
bd_3.text(center(bt_3, ticket, fonts[0]), ticket,
font=fonts[0], fill=b)
sd_1.text(center(st_1, link, fonts[2]), link, font=fonts[2], fill=b)
sd_2.text(center(st_2, border, fonts[2]), border, font=fonts[2], fill=b)
sd_3.text(center(st_3, border, fonts[2]), border, font=fonts[2], fill=b)
sd_4.text(center(st_4, office, fonts[2]), office, font=fonts[2], fill=b)
sd_5.text(center(st_5, task, fonts[2]), task, font=fonts[2], fill=b)
sd_6.text(center(st_6, date, fonts[2]), date, font=fonts[2], fill=b)
sd_7.text(center(st_7, ahead, fonts[2]), ahead, font=fonts[2], fill=b)
sd_8.text(center(st_8, cuti, fonts[2]), cuti, font=fonts[2], fill=b)
tts = 0
mt.paste(bt_1, (0, 0))
tts += bt_1.size[1]
mt.paste(bt_2, (0, tts))
tts += bt_2.size[1]
mt.paste(st_1, (0, tts))
tts += st_1.size[1]
mt.paste(st_2, (0, tts))
tts += st_2.size[1]
mt.paste(bt_3, (0, tts))
tts += bt_3.size[1]
mt.paste(st_3, (0, tts))
tts += st_3.size[1]
mt.paste(st_4, (0, tts))
tts += st_4.size[1]
mt.paste(st_8, (0, tts))
tts += st_8.size[1]
mt.paste(st_7, (0, tts))
tts += st_7.size[1]
mt.paste(st_5, (0, tts))
tts += st_5.size[1]
mt.paste(st_6, (0, tts))
sffs = []
sfs = [
f'{uuid.uuid4()}'.replace('-', '') + '.jpg',
f'{uuid.uuid4()}'.replace('-', '') + '.txt'
]
for f in sfs:
sffs.append(path.join(getcwd(), f))
mt.save(sfs[0], format="JPEG")
p = Dummy()
p.image(sfs[0], fragment_height=tt, high_density_vertical=True)
p.cut()
f = open(sfs[1], 'wb+')
f.write(p.output)
p.close()
f.close()
if kwargs.get('windows'):
system(f'print /D:\\\{host}\\"{pname}" "{sffs[1]}"')
elif kwargs.get('unix'):
system(f'lp -d "{pname}" -o raw "{sffs[1]}"')
for f in sffs:
if path.isfile(f):
remove(f)
def plot_pollster_house_effects(self, samples, hebrew=True):
"""
Plot the house effects of each pollster per party.
"""
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.ticker as ticker
from bidi import algorithm as bidialg
house_effects = samples.transpose(2, 1, 0)
fe = self.forecast_model
actual_pollsters = [
i for i in fe.dynamics.pollster_mapping.items() if i[1] is not None
]
pollster_ids = [
fe.pollster_ids[pollster]
for _, pollster in sorted(actual_pollsters, key=lambda i: i[1])
]
plots = []
for i, party in enumerate(fe.party_ids):
def pollster_label(pi, pollster_id):
perc = '%.2f %%' % (100 * house_effects[i][pi].mean())
if hebrew and len(
fe.config['pollsters'][pollster_id]['hname']) > 0:
label = perc + ' :' + bidialg.get_display(
fe.config['pollsters'][pollster_id]['hname'])
else:
label = fe.config['pollsters'][pollster_id][
'name'] + ': ' + perc
return label
cpalette = sns.color_palette("cubehelix", len(pollster_ids))
patches = [
mpatches.Patch(color=cpalette[pi],
label=pollster_label(pi, pollster))
for pi, pollster in enumerate(pollster_ids)
]
fig, ax = plt.subplots(figsize=(10, 2))
fig.set_facecolor('white')
legend = fig.legend(handles=patches, loc='best', ncol=2)
if hebrew:
for col in legend._legend_box._children[-1]._children:
for c in col._children:
c._children.reverse()
col.align = "right"
ax.set_title(
bidialg.get_display(fe.parties[party]['hname']
) if hebrew else fe.parties[party]['name'])
for pi, pollster_house_effects in enumerate(house_effects[i]):
sns.kdeplot(100 * pollster_house_effects,
shade=True,
ax=ax,
color=cpalette[pi])
ax.xaxis.set_major_formatter(ticker.PercentFormatter(decimals=1))
ax.yaxis.set_major_formatter(ticker.PercentFormatter(decimals=1))
plots += [ax]
fig.text(.5,
1.05,
bidialg.get_display('הטיית הסוקרים')
if hebrew else 'House Effects',
ha='center',
fontsize='xx-large')
fig.text(.5,
.05,
'Generated using pyHoshen © 2019 - 2021',
ha='center')
def snapshot(self, dt):
# Read the frame at button press
ret, frame = self.capture.read()
# Stop the video
self.capture.release()
ArrayOCR, indexLangFrom, ArrayText, indexLangTo, LangTo = self.submit(
self)
if ret:
# Select region of interest
img_raw = frame.copy()
ROIs = cv2.selectROIs("Select Region of interest", img_raw)
print(ROIs)
for rect in ROIs:
x1 = rect[0] # x
y1 = rect[1] # y
x2 = rect[2] # width
y2 = rect[3] # height
# crop roi from original image
img_crop = img_raw[y1:y1 + y2, x1:x1 + x2]
cv2.destroyAllWindows()
# Show cropped image after grayscale
gray = cv2.cvtColor(img_crop, cv2.COLOR_BGR2GRAY)
cv2.imshow('gray', gray)
# Pytesseract detect image
options = "-l {} --psm {} --oem {}".format(
ArrayOCR[indexLangFrom], args["psm"], args["oem"])
text = pytesseract.image_to_string(gray, config=options)
# show the original OCR'd text
print("ORIGINAL")
print("========")
print(text)
print("")
# Google translator translate the OCR'd text
translator = Translator()
# Keep repeating if api is banned by Google
while True:
try:
textTranslate = translator.translate(
text,
src=ArrayText[indexLangFrom],
dest=ArrayText[indexLangTo])
break
except Exception as e:
translator = Translator()
print('Translation')
print(f'source: {textTranslate.src}')
print(f'Destination: {textTranslate.dest}')
print(f'{textTranslate.text}')
print("")
# Display the image on Kivy at the same coordinates of the cropped image
if LangTo == 'arabic':
reshaped_text = arabic_reshaper.reshape(textTranslate.text)
text = get_display(reshaped_text)
al = "right"
font_path = "arial.ttf"
font = ImageFont.truetype(font_path, 40)
img_pil = ImagePIL.fromarray(img_raw)
draw = ImageDraw.Draw(img_pil)
draw.rectangle((x1, y1, x1 + x2, y1 + y2),
fill=(173, 171, 170))
draw.text((x1, y1),
text,
font=font,
align=al,
fill=(255, 0, 0))
img_raw = np.array(img_pil)
else:
text = textTranslate.text
al = "left"
font_path = "arial.ttf"
font = ImageFont.truetype(font_path, 40)
img_pil = ImagePIL.fromarray(img_raw)
draw = ImageDraw.Draw(img_pil)
draw.rectangle((x1, y1, x1 + x2, y1 + y2),
fill=(173, 171, 170))
draw.text((x1, y1),
text,
font=font,
align=al,
fill=(255, 0, 0))
img_raw = np.array(img_pil)
buf2 = cv2.flip(img_raw, 0)
buf = buf2.tostring()
texture2 = Texture.create(size=(img_raw.shape[1],
img_raw.shape[0]),
colorfmt='bgr')
texture2.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
self.img1.texture = texture2
def plot_coalitions(self,
bader_ofer,
coalitions=None,
day=0,
min_mandates_for_coalition=61,
stable_mandates_for_coalition=65,
hebrew=True,
coalitions_shape=None,
subtitle=''):
"""
Plot the resulting mandates of the coalitions and their distributions.
"""
from bidi import algorithm as bidialg
fe = self.forecast_model
if coalitions is None:
coalitions = fe.config['coalitions']
num_coalitions = len(coalitions)
coalitions_matrix = np.zeros([num_coalitions, fe.num_parties],
dtype='bool')
for i, (coalition, config) in enumerate(coalitions.items()):
for party in config['parties']:
party_index = fe.party_ids.index(party)
coalitions_matrix[i][party_index] = 1
bo_plot = bader_ofer.transpose(1, 2, 0)[day]
coalitions_mandates = coalitions_matrix[:, :, None] * bo_plot[
None, :, :].astype('int64')
for i, (coalition, config) in enumerate(coalitions.items()):
if 'subsets' in config:
for party, subset_config in config['subsets'].items():
party_index = fe.party_ids.index(party)
cum_members = np.asarray([
sum(j <= i for j in subset_config['members'])
for i in range(1 + max(subset_config['members']))
])
coalitions_mandates[i][party_index] = cum_members[
coalitions_mandates[i][party_index]]
coalitions_bo = coalitions_mandates.sum(axis=1)
if coalitions_shape is None:
coalitions_shape = (1, num_coalitions)
fig, plots = plt.subplots(coalitions_shape[0],
coalitions_shape[1],
figsize=(5 * coalitions_shape[1],
5 * coalitions_shape[0]))
fig.set_facecolor('white')
xlim_dists = []
ylim_height = []
colors = [
'#ff0000', # 57 = red
'#ff3d00', # 58
'#ff7900', # 59
'#ffb600', # 60
'#fff200', # 61 = yellow
'#c7db00', # 62
'#8fc400', # 63
'#56ad00', # 64
'#1e9600', # 65 = green
]
coalition_names = sorted([
bidialg.get_display(config['hname']) if hebrew else config['name']
for config in coalitions.values()
],
key=lambda p: p[::-1] if hebrew else p,
reverse=hebrew)
for i, (coalition, config) in enumerate(coalitions.items()):
name = bidialg.get_display(
config['hname']) if hebrew else config['name']
coalition_index = coalition_names.index(name)
if coalitions_shape[0] > 1:
plot = plots[coalition_index // coalitions_shape[1],
coalition_index % coalitions_shape[1]]
else:
plot = plots[coalition_index]
title = plot.set_title(name, va='bottom', y=-0.2, fontsize='large')
def get_party_name(party):
namevar = 'hname' if hebrew else 'name'
name = fe.parties[party][namevar]
if 'subsets' in config and party in config['subsets']:
name += ' (%s)' % config['subsets'][party][namevar]
return bidialg.get_display(name) if hebrew else name
party_names = [
get_party_name(party) for party in config['parties']
]
plot.text(0.5,
-0.2,
'\n'.join(
sorted(party_names,
key=lambda p: p[::-1] if hebrew else p)),
ha='center',
va='top',
fontsize='small',
transform=plot.transAxes)
mandates_count = np.unique(coalitions_bo[i], return_counts=True)
bars = plot.bar(mandates_count[0],
100 * mandates_count[1] / len(coalitions_bo[i]))
for mandates, bar in zip(mandates_count[0], bars):
cindex = int(min(8, max(0, mandates - 57)))
bar.set_color(colors[cindex])
xticks = []
max_start = 0
if 0 in mandates_count[0]:
xticks += [0]
max_start = 1
mean_mandates = coalitions_bo[i].mean()
if len(mandates_count[1]) > max_start:
mean_index = np.where(
mandates_count[0] == int(np.round(mean_mandates)))
mean_index = mean_index[0][0]
mean_rect = bars[mean_index]
xticks += [mandates_count[0][mean_index]]
num_minimum = len(
np.where(coalitions_bo[i] >= min_mandates_for_coalition)[0])
num_stable = len(
np.where(coalitions_bo[i] >= stable_mandates_for_coalition)[0])
perc_text = plot.text(
mean_rect.get_x() + mean_rect.get_width() / 2.0,
mean_rect.get_y() + mean_rect.get_height() / 3.0,
'%.1f%%' % (100 * num_minimum / coalitions_bo[i].shape[0]),
ha='center',
va='top',
fontsize='xx-large',
fontweight='bold')
perc_text.set_path_effects(
[pe.withStroke(linewidth=4, foreground='w', alpha=0.7)])
perc_title = plot.text(
mean_rect.get_x() + mean_rect.get_width() / 2.0,
mean_rect.get_y() + mean_rect.get_height() / 3.0,
bidialg.get_display('%d ומעלה:' % min_mandates_for_coalition)
if hebrew else '%d and Above:' % min_mandates_for_coalition,
ha='center',
va='bottom',
fontsize='medium',
fontweight='bold')
perc_title.set_path_effects(
[pe.withStroke(linewidth=4, foreground='w', alpha=0.7)])
perc_text = plot.text(
mean_rect.get_x() + mean_rect.get_width() / 2.0,
mean_rect.get_y() + 2 * mean_rect.get_height() / 3.0,
'%.1f%%' % (100 * num_stable / coalitions_bo[i].shape[0]),
ha='center',
va='top',
fontsize='xx-large',
fontweight='bold')
perc_text.set_path_effects(
[pe.withStroke(linewidth=4, foreground='w', alpha=0.7)])
perc_title = plot.text(
mean_rect.get_x() + mean_rect.get_width() / 2.0,
mean_rect.get_y() + 2 * mean_rect.get_height() / 3.0,
bidialg.get_display('%d ומעלה:' %
stable_mandates_for_coalition)
if hebrew else '%d and Above:' % stable_mandates_for_coalition,
ha='center',
va='bottom',
fontsize='medium',
fontweight='bold')
perc_title.set_path_effects(
[pe.withStroke(linewidth=4, foreground='w', alpha=0.7)])
plot.set_xticks(xticks)
xlim = plot.get_xlim()
xlim_dists += [xlim[1] - xlim[0] + 1]
ylim_height += [plot.get_ylim()[1]]
plot.grid(False)
plot.tick_params(axis='y',
which='both',
left=False,
labelleft=False)
plot.set_facecolor('white')
for s in plot.spines.values():
s.set_visible(False)
xlim_side = max(xlim_dists) / 2
for i in range(num_coalitions):
xlim = plot.get_xlim()
xlim_center = (xlim[0] + xlim[1]) / 2
plot.set_xlim(xlim_center - xlim_side, xlim_center + xlim_side)
plot.set_ylim(top=max(ylim_height))
title = 'קואליציות' if hebrew else 'Coalitions'
if subtitle != '':
title += ' - ' + subtitle
if hebrew:
title = bidialg.get_display(title)
fig.text(.5, 1.05, title, ha='center', fontsize='xx-large')
if fe.house_effects_model is not None:
fig.text(.5,
1.,
self.house_effects_model_title(hebrew),
ha='center',
fontsize='small')
fig.figimage(self.create_logo(),
fig.bbox.xmax / 2 + 100,
fig.bbox.ymax - 0,
zorder=1000)
def word_process(self, x):
reshaped_text = arabic_reshaper.reshape(x)
return get_display(reshaped_text)
def format_text(self, text):
if self.font_type in ['normal', 'handwritten']:
text = unidecode(text)
elif self.font_type == 'arabic':
text = TRANSLATOR.translate(text, src='en', dest='ar').text
text = get_display(arabic_reshaper.reshape(text))
elif self.font_type == 'chinese':
text = TRANSLATOR.translate(text, src='en', dest='zh-CN').text
else:
raise NotImplementedError
width, height = self.width - 2 * self.padding, self.height - 2 * self.padding
text = (text.upper() if self.uppercase else text).strip()
if self.text_type == 'word':
word_as_number = choice([True, False])
if word_as_number:
n_letter = randint(1, 5)
result_text = str(randint(0, 10**n_letter - 1))
else:
words = text.split(' ')
result_text = rand_choice(words)
iteration = 1
while (not str.isalnum(result_text) or len(result_text) < 1) and iteration < 40:
result_text = rand_choice(words)
iteration += 1
if not str.isalnum(result_text) or len(result_text) < 1:
result_text = words[0][:randint(4, 10)]
line_width = self.font.getsize(result_text)[0]
while line_width > width and len(result_text) > 2:
result_text = result_text[:-1]
line_width = self.font.getsize(result_text)[0]
else:
max_lines = 1 if self.uniline else self.n_max_lines
result_text, lines = '', ''
text_height, cur_idx, n_lines = 0, 0, -1
while text_height <= height:
result_text = lines
n_lines += 1
line = text[cur_idx:].lstrip()
cur_idx += len(text[cur_idx:]) - len(line) # adjust cur_idx if stripped
if len(line) == 0 or n_lines == max_lines:
break
line_width = self.font.getsize(line)[0]
avg_char_width = line_width / len(line)
if line_width > width:
index = int(width / avg_char_width) + 10 # take larger slice in case of small characters
cut = max(line[:index].rfind(' '), line.find(' ')) # in case no space found in slice (small width)
line = line[:cut].strip()
line_width = self.font.getsize(line)[0]
while line_width > width:
if ' ' in line: # remove word by word
line = line[:line.rfind(' ')].strip()
else: # remove character by character
line = line[:-1]
line_width = self.font.getsize(line)[0]
cur_idx += len(line) + 1
if self.justified:
w_space = self.font.getsize(' ')[0]
n_spaces = line.count(' ')
n_spaces_to_add = (width - line_width) // w_space
if n_spaces > 0 and n_spaces_to_add > 0:
q, r = n_spaces_to_add // n_spaces + 1, n_spaces_to_add % n_spaces
if q < 5:
if q > 1:
line = line.replace(' ', q * ' ')
pos = 0
while r > 0:
space_idx = line[pos:].find(' ') + pos
line = line[:space_idx] + ' ' + line[space_idx:]
pos = space_idx + q + 1
r -= 1
lines = '{}\n{}'.format(lines, line) if lines else line
text_height = self.font.getsize_multiline(lines, spacing=self.spacing)[1]
if '\n' in result_text and self.justified: # we dont want to justify the last line
result_text, last_line = result_text.rsplit('\n', 1)
last_line = ' '.join(last_line.split())
result_text = '{}\n{}'.format(result_text, last_line)
content_width, content_height = self.font.getsize_multiline(result_text, spacing=self.spacing)
content_width += 2 * self.padding
content_height += 2 * self.padding
return result_text, content_width, content_height
def _bidi_transpose(self, text):
if self.settings.bidi_support and HAVE_BIDI:
return get_display(text)
return text
def _get_text(im):
with open(os.path.splitext(im)[0] + '.gt.txt', 'r') as fp:
return get_display(fp.read())
async def profile(self, ctx, user: discord.Member = None):
if user == None:
user = ctx.author
ava = user.avatar_url_as(size=128, format='png') # save avatar user
data = BytesIO(await ava.read()) # None
pfp = Image.open(data)
pfp = pfp.resize((94, 94))
bigsize = (pfp.size[0] * 3, pfp.size[1] * 3)
mask = Image.new('L', bigsize, 0)
draw = ImageDraw.Draw(mask)
draw.ellipse((0, 0) + bigsize, fill=255)
mask = mask.resize(pfp.size, Image.ANTIALIAS)
pfp.putalpha(mask)
output = ImageOps.fit(pfp, mask.size, centering=(0.5, 0.5))
output.putalpha(mask)
output.save('./img/output.png')
background = Image.open('./img/Profile_.png')
draw = ImageDraw.Draw(background) # draw img
background.paste(pfp, (15, 14), pfp)
font = ImageFont.truetype("font/Arial.ttf", size=24) # font all text
shadow_color = "white" # shadow color all text
stroke_width = 1 # stroke width
color_stroke = f"{user.color}" # color stroke
username = user.name # get user name
user_tag = "#" + user.discriminator # get user tag
join_at = user.created_at.strftime("%Y/%m/%d") # get join at
user_id = user.id # get user id
reshaped_text = arabic_reshaper.reshape(username)
bidi_text = get_display(reshaped_text)
text = self.cleanWord(bidi_text)
if len(text) > 9:
text = text[:10] + "..."
elif len(text) > 8:
text = text[:9] + ".."
elif len(text) > 7:
text = text[:8] + "."
# fonts and size name
# draw.text(
# [142, 129],
# text, # add arabic
# font=font,
# stroke_fill="black"
# )
draw.text(
[141, 128],
text, # add arabic
font=font,
fill=shadow_color,
stroke_width=stroke_width,
stroke_fill=color_stroke
)
# fonts and size tag
draw.text(
[141, 173],
user_tag,
font=font,
fill=shadow_color,
stroke_width=stroke_width,
stroke_fill=color_stroke)
# fonts and size join at
draw.text(
[141, 217],
join_at,
font=font,
fill=shadow_color,
stroke_width=stroke_width,
stroke_fill=color_stroke)
# fonts and size user id
draw.text(
[46, 259],
str(user_id),
font=font,
fill=shadow_color,
stroke_width=stroke_width,
stroke_fill=color_stroke)
background.save('./img/profile.png') # save img
await ctx.send(file=discord.File(f"./img/profile.png")) # send profile img
os.remove("./img/profile.png")
os.remove("./img/output.png")
def _get_text(im):
with open(os.path.splitext(im)[0] + '.gt.txt', 'r') as fp:
return get_display(fp.read())
def GenerateNeighborhoodIndexMap(NeighborhoodIndex, ShapeFileLocation,
FileLocation, FileName):
fig, ax = plt.subplots(1, figsize=(14, 20))
minval = np.min(NeighborhoodIndex[:, 0].astype(float))
maxval = np.max(NeighborhoodIndex[:, 0].astype(float))
norm = mpl.colors.Normalize(minval, maxval)
#print minval,maxval
cmap = cm.RdPu
m = cm.ScalarMappable(norm=norm, cmap=cmap)
temp = []
ax.set_title('Estimated Value', fontsize=20)
with collection(ShapeFileLocation, "r", encoding='utf8') as input:
# schema = input.schema.copy()
schema = {'geometry': 'Polygon', 'properties': {'NEIGHBORH1': 'str'}}
for point in input:
BoolPass = True
ShapeFileDistrict = point['properties']['District_N']
ShapeFileMunicipality = point['properties']['Municipali']
temp.append([ShapeFileDistrict, ShapeFileMunicipality])
try:
select = np.logical_and(
NeighborhoodIndex[:, 1] == ShapeFileDistrict,
NeighborhoodIndex[:, 2] == ShapeFileMunicipality)
EstVal = float(NeighborhoodIndex[select][0][0])
BoolPass = True
except Exception as e:
BoolPass = False
pass
if BoolPass:
poly = Polygon(np.array(point['geometry']['coordinates'][0]))
x, y = poly.exterior.xy
ring_patch = PolygonPatch(poly,
fc=m.to_rgba(EstVal),
ec=m.to_rgba(EstVal),
alpha=0.9,
zorder=2)
ax.add_patch(ring_patch)
ax.plot(x,
y,
color='#000000',
alpha=0.85,
linewidth=1,
solid_capstyle='butt',
zorder=2)
x, y = poly.centroid.wkt.replace(')',
'').replace('(', '').replace(
'POINT ', '').split(' ')
rx = float(x)
ry = float(y)
cx = rx # + ring_patch.get_width()/2.0
cy = ry # + ring_patch.get_height()/2.0
reshaped_text = arabic_reshaper.reshape(
point['properties'][u'District_N'])
artext = get_display(reshaped_text)
ax.annotate(artext, (cx, cy),
color='black',
weight='bold',
fontsize=4,
ha='center',
va='center')
else:
poly = Polygon(np.array(point['geometry']['coordinates'][0]))
x, y = poly.exterior.xy
ring_patch = PolygonPatch(poly,
fc='#FFFFFF',
ec='#FFFFFF',
alpha=0.9,
zorder=2)
ax.add_patch(ring_patch)
ax.plot(x,
y,
color='#000000',
alpha=0.85,
linewidth=1,
solid_capstyle='butt',
zorder=2)
#Add the neighborhood name
x, y = poly.centroid.wkt.replace(')',
'').replace('(', '').replace(
'POINT ', '').split(' ')
rx = float(x)
ry = float(y)
cx = rx # + ring_patch.get_width()/2.0
cy = ry # + ring_patch.get_height()/2.0
reshaped_text = arabic_reshaper.reshape(
point['properties'][u'District_N'])
artext = get_display(reshaped_text)
ax.annotate(artext, (cx, cy),
color='black',
weight='bold',
fontsize=4,
ha='center',
va='center')
#Get the min and max values of the method
plt.xticks([])
plt.yticks([])
plt.savefig(FileLocation + '/' + FileName + '.png',
bbox_inches='tight',
dpi=300)
def getPicture(poem,poet):
allBGs = glob.glob('/home/ali/Pictures/background/*.*')
selected = randint(0,len(allBGs)-1)
pic = allBGs[selected]
img = Image.open(pic)
img= img.resize((800,300))
offset = 200;
draw = ImageDraw.Draw(img)
sentences = poem
maxSize = 0
boundingSize = (550,300)
for sent in sentences:
if(len(sent) > maxSize):
maxSize = len(sent)
maxSent = sent
t1 = arabic_reshaper.reshape(maxSent.decode('utf-8'))
maxSent = get_display(t1)
fontSize =2
fontAddress = "/home/ali/Fonts/Mj_Sandbad.ttf"
while(1):
font = ImageFont.truetype(fontAddress, fontSize)
draw.setfont(font)
width = draw.textsize(maxSent)[0]
height = len(sentences)* draw.textsize(maxSent)[1]
if(width<boundingSize[0] and height < boundingSize[1]):
fontSize = fontSize+1
continue
else:
break
font = ImageFont.truetype(fontAddress, fontSize-1)
draw.setfont(font)
i=0
W,H = boundingSize
color = (255,255,255)
for sent in sentences:
t1 = arabic_reshaper.reshape(sent.decode('utf-8'))
text = get_display(t1)
size = draw.textsize(text)
## draw.text((x, y),"Sample Text",(r,g,b))
draw.text((W/2-size[0]/2+offset, H/2-len(sentences)*size[1]/2-size[1]/4+i*size[1]),text, color)
i=i+1
font = ImageFont.truetype(fontAddress, 25)
draw.setfont(font)
t1 = arabic_reshaper.reshape(poet.decode('utf-8'))
ppoet = get_display(t1)
size = draw.textsize(ppoet)
position = (W/2-size[0]/2,H-size[1]-20)
#pixel = img.getpixel(position)
#hsv = colorsys.rgb_to_hsv(float(pixel[0])/255,float(pixel[1])/255,float(pixel[2])/255)
#rgb = colorsys.hsv_to_rgb(((hsv[0]-.5)*360%360)/360,1-hsv[1],1-hsv[2])
rgb = (1,1,1)
draw.text(position,ppoet,(int(255*rgb[0]),int(255*rgb[1]),int(255*rgb[2])))
avatar = getAvatar(poet)
avatar = avatar.resize([offset,offset])
img.paste(avatar, (0, img.size[1]-offset), avatar)
return img
def GenerateCorrelationPlot(NeighborhoodIndex, CompanyEstimations, CompanyName,
FileLocation, FileName):
RankCorr = []
for Neigh in NeighborhoodIndex:
for Neigh2 in CompanyEstimations:
if Neigh[1] == Neigh2[1] and Neigh[2] == Neigh2[2]:
RankCorr.append([Neigh[0], Neigh2[0], Neigh[1], Neigh[2]])
break
RankCorr = np.array(RankCorr)
Errors = np.abs(RankCorr[:, 0].astype(float) -
RankCorr[:, 1].astype(float))
Error = np.mean(Errors)
PercentageDiff = []
for item in RankCorr[:, [0, 1]].astype(float):
PercentageDiff.append(np.abs((item[0] - item[1]) / item[1]))
#compute Rsquared
dividend = np.sum(
np.power(RankCorr[:, 1].astype(float) - RankCorr[:, 0].astype(float),
2))
divisor = np.sum(
np.power(
RankCorr[:, 1].astype(float) -
np.mean(RankCorr[:, 1].astype(float)), 2))
Rsquared = 1 - (dividend / divisor)
#print PercentageDiff
CreateDistributionPlot(Errors, CompanyName, FileLocation, FileName,
np.linspace(0, 50000, 50))
CreateDistributionPlot(PercentageDiff, CompanyName, FileLocation,
FileName + '_PercentageDiff', np.linspace(0, 1, 20))
r, pvalue = spearmanr(RankCorr[:, 0].astype(float),
RankCorr[:, 1].astype(float))
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
ax.scatter(RankCorr[:, 0].astype(float), RankCorr[:, 1].astype(float))
ax.set_xlabel('ECDF estimation', fontsize=20)
ax.set_ylabel(CompanyName, fontsize=20)
for i, txt in enumerate(RankCorr[:, 2]):
reshaped_text = arabic_reshaper.reshape(unicode(txt))
artext = get_display(reshaped_text)
ax.annotate(artext, (RankCorr[i, 0], RankCorr[i, 1]), fontsize=6)
MaxECDF = np.max(RankCorr[:, 0].astype(float))
MaxOther = np.max(RankCorr[:, 1].astype(float))
if MaxECDF > MaxOther:
ax.set_xlim([0, MaxECDF])
ax.set_ylim([0, MaxECDF])
ax.plot(range(int(MaxECDF)), range(int(MaxECDF)), '--', color='black')
else:
ax.set_xlim([0, MaxOther])
ax.set_ylim([0, MaxOther])
ax.plot(range(int(MaxOther)),
range(int(MaxOther)),
'--',
color='black')
ax.set_title(
str((r, pvalue)) + '\n' + 'MAE:' + str(Error) + '\n R_squared' +
str(Rsquared))
plt.savefig(FileLocation + '/' + FileName + '.png',
bbox_inches='tight',
dpi=300)
return r, pvalue
def __unicode__(self):
return algorithm.get_display(self.username)
def GenerateNeighborhoodIndexBarChart(NeighborhoodIndex, FileLocation,
FileName):
fig = plt.figure(figsize=(10, 20))
ax = fig.add_subplot(111)
Sorted = np.argsort(NeighborhoodIndex[:, 0].astype(float) * 1)
#Draw the colored bars
tempr = NeighborhoodIndex[Sorted][:, 0].astype(float)
count = 0
barlist = ax.barh(range(len(tempr[tempr < 1000])),
tempr[tempr < 1000],
color='g',
alpha=0.7,
align='center')
count += len(tempr[tempr < 1000])
select = np.logical_and(tempr < 2000, tempr >= 1000)
barlist = ax.barh(range(len(tempr[tempr < 1000]),
len(tempr[tempr < 2000])),
tempr[select],
color='gray',
alpha=0.7,
align='center')
count += len(tempr[select])
select = np.logical_and(tempr < 3000, tempr >= 2000)
barlist = ax.barh(range(len(tempr[tempr < 2000]),
len(tempr[tempr < 3000])),
tempr[select],
color='r',
alpha=0.7,
align='center')
count += len(tempr[select])
select = np.logical_and(tempr < 100000, tempr >= 3000)
barlist = ax.barh(range(len(tempr[tempr < 3000]),
len(tempr[tempr < 100000])),
tempr[select],
color='steelblue',
align='center')
count += len(tempr[select])
#Add the neighborhoods names
Neigh_names_new = []
for item in NeighborhoodIndex[Sorted][:, 1]:
reshaped_text = arabic_reshaper.reshape(unicode(item))
artext = get_display(reshaped_text)
Neigh_names_new.append(artext)
ax.yaxis.set_ticks(np.arange(0, len(Neigh_names_new), 1))
ax.set_yticklabels(Neigh_names_new, fontsize=8)
#Edit the chart details
ax.legend(['0 - 1000', '1000 - 2000', '2000 - 3000', '>3000'],
fontsize=20,
loc=7)
#ax.grid()
ax.xaxis.grid(True, which='major')
ax.set_ylim([0, len(NeighborhoodIndex[:, 0])])
ax.set_xlim([0, 9000])
ax.set_xlabel('Estimated Value', fontsize=20)
ax.set_ylabel('Neighborhoods', fontsize=20)
plt.savefig(FileLocation + '/' + FileName + '.png',
bbox_inches='tight',
dpi=300)
