def setLanguageXX(self, xx):
# greek officially ISO639-1 is 'el' , but opensubtitles is buggy
if xx == 'gr':
xx = 'el'
self._languageXX = xx
self._languageXXX = languages.xx2xxx(xx)
self._languageName = languages.xx2name(xx)
def setLanguageXX(self, xx):
# greek officially ISO639-1 is 'el' , but opensubtitles is buggy
if xx == 'gr':
xx = 'el'
self._languageXX = xx
self._languageXXX = languages.xx2xxx(xx)
self._languageName = languages.xx2name(xx)
def AutoDetectLangFromFileName(self):
all_langs = []
xxx_lang = ""
for sub in self._subs:
if sub:
lang = sub.getLanguage()
if lang == None:
lang = Subtitle.GetLangFromFilename(sub.getFilePath())
if len(lang) == 2 and lang in languages.ListAll_xx():
all_langs.append(languages.xx2xxx(lang))
elif len(lang) == 3 and lang in languages.ListAll_xxx():
all_langs.append(lang)
else:
all_langs.append(lang)
max = 0
max_lang = ""
for lang in all_langs:
if all_langs.count(lang) > max:
max = all_langs.count(lang)
max_lang = lang
xxx_lang = max_lang
log.debug("Majoritary Language Autodetected by filename = " +
str(xxx_lang))
if xxx_lang:
self._main.language_updated.emit(xxx_lang, "filename")
def do_matching(self, videos, subtitles):
if self.options.logging > logging.DEBUG and self.options.verbose:
progress = progressbar.ProgressBar(
widgets=conf.Terminal.progress_bar_style, maxval=len(videos)).start()
for i, video in enumerate(videos):
if self.options.logging > logging.DEBUG and self.options.verbose:
progress.update(i + 1)
self.log.debug("Processing %s..." % video.getFileName())
possible_subtitle = Subtitle.AutoDetectSubtitle(
video.getFilePath())
#self.log.debug("possible subtitle is: %s"% possible_subtitle)
sub_match = None
for subtitle in subtitles:
sub_match = None
if possible_subtitle == subtitle.getFilePath():
sub_match = subtitle
self.log.debug("Match found: %s" % sub_match.getFileName())
break
if sub_match:
sub_lang = Subtitle.AutoDetectLang(sub_match.getFilePath())
sub_match.setLanguage(Languages.name2xxx(sub_lang))
video.addSubtitle(sub_match)
if self.options.logging > logging.DEBUG and self.options.verbose:
progress.finish()
def AutoDetectLangFromFileName(self):
all_langs = []
xxx_lang = ""
for sub in self._subs:
if sub:
lang = sub.getLanguage()
if lang == None:
lang = Subtitle.GetLangFromFilename(sub.getFilePath())
if len(lang) == 2 and lang in languages.ListAll_xx():
all_langs.append(languages.xx2xxx(lang))
elif len(lang) == 3 and lang in languages.ListAll_xxx():
all_langs.append(lang)
else:
all_langs.append(lang)
max = 0
max_lang = ""
for lang in all_langs:
if all_langs.count(lang) > max:
max = all_langs.count(lang)
max_lang = lang
xxx_lang = max_lang
log.debug(
"Majoritary Language Autodetected by filename = " + str(xxx_lang))
if xxx_lang:
self._main.language_updated.emit(xxx_lang, "filename")
def do_matching(self, videos, subtitles):
if self.options.logging > logging.DEBUG and self.options.verbose:
progress = progressbar.ProgressBar(
widgets=conf.Terminal.progress_bar_style,
maxval=len(videos)).start()
for i, video in enumerate(videos):
if self.options.logging > logging.DEBUG and self.options.verbose:
progress.update(i + 1)
self.log.debug("Processing %s..." % video.getFileName())
possible_subtitle = Subtitle.AutoDetectSubtitle(
video.getFilePath())
#self.log.debug("possible subtitle is: %s"% possible_subtitle)
sub_match = None
for subtitle in subtitles:
sub_match = None
if possible_subtitle == subtitle.getFilePath():
sub_match = subtitle
self.log.debug("Match found: %s" % sub_match.getFileName())
break
if sub_match:
sub_lang = Subtitle.AutoDetectLang(sub_match.getFilePath())
sub_match.setLanguage(Languages.name2xxx(sub_lang))
video.addSubtitle(sub_match)
if self.options.logging > logging.DEBUG and self.options.verbose:
progress.finish()
def __init__(self, parent, main):
QDialog.__init__(self, parent)
self.ui = Ui_PreferencesDialog()
self.ui.setupUi(self)
self._main = main
settings = QSettings()
# OPTIONS events
self.ui.optionsButtonApplyChanges.clicked.connect(
self.onOptionsButtonApplyChanges)
self.ui.optionsButtonCancel.clicked.connect(
self.onOptionsButtonCancel)
self.ui.optionButtonChooseFolder.clicked.connect(
self.onOptionButtonChooseFolder)
self.ui.optionDownloadFolderPredefined.clicked.connect(
self.onOptionDownloadFolderPredefined)
self.ui.optionVideoAppChooseLocation.clicked.connect(
self.onOptionVideoAppChooseLocation)
self.ui.helpTranslateButton.clicked.connect(
self.onOptionHelpTranslateButton)
self.onOptionDownloadFolderPredefined()
self.filterLanguages = {}
self.ui.optionDefaultUploadLanguage.addItem(_("<AutoDetect>"), "")
for num, lang in enumerate(languages.LANGUAGES):
lang_xxx = lang["SubLanguageID"]
self.ui.optionDefaultUploadLanguage.addItem(
_(lang["LanguageName"]), lang_xxx)
# Adding checkboxes for the Search...Filter by ...
self.filterLanguages[lang_xxx] = QCheckBox(
_(lang["LanguageName"]), self.ui.scrollAreaWidgetContents)
if num % 4 == 1:
self.ui.optionFilterLangLayout_1.addWidget(
self.filterLanguages[lang_xxx])
elif num % 4 == 2:
self.ui.optionFilterLangLayout_2.addWidget(
self.filterLanguages[lang_xxx])
elif num % 4 == 3:
self.ui.optionFilterLangLayout_3.addWidget(
self.filterLanguages[lang_xxx])
else:
self.ui.optionFilterLangLayout_4.addWidget(
self.filterLanguages[lang_xxx])
for lang_locale in self._main.interface_langs:
languageName = languages.locale2name(lang_locale)
if not languageName:
languageName = lang_locale
self.ui.optionInterfaceLanguage.addItem(
_(languageName), lang_locale)
self.ui.optionDefaultUploadLanguage.adjustSize()
self.ui.optionInterfaceLanguage.adjustSize()
self.readOptionsSettings(settings)
self.ui.optionInterfaceLanguage.currentIndexChanged.connect(
self.onOptionInterfaceLanguage)
def core_before_request():
""" To run before each request"""
my = None
g.db = db
languages_object = Languages()
g.languages = languages_object.get_languages(0)
get_headers()
# Check that user has login
if 'user_id' in session:
# get user_id from session
user_id = session['user_id']
# get the user's personal data.
my = g.db.users.find_one({ '_id' : ObjectId(user_id) })
# If user_id not exist in the user list g.my is None
if my:
g.my = my
# get user language
g.lan = g.my['lan']
g.language = g.languages[g.lan]
get_hash_admin()
def __init__(self, parent, user_locale):
QtGui.QDialog.__init__(self)
self.ui = Ui_ChooseLanguageDialog()
self.ui.setupUi(self)
self._main = parent
settings = QSettings()
QObject.connect(self.ui.languagesList, SIGNAL("activated(QModelIndex)"), self.onOkButton)
QObject.connect(self.ui.OKButton, SIGNAL("clicked(bool)"), self.onOkButton)
for lang_locale in self._main.interface_langs:
languageName = Languages.locale2name(lang_locale)
if not languageName:
languageName = lang_locale
item = QListWidgetItem(languageName)
item.setData(Qt.UserRole, QVariant(lang_locale))
self.ui.languagesList.addItem(item)
try:
if lang_locale == user_locale:
self.ui.languagesList.setCurrentItem(item,QItemSelectionModel.ClearAndSelect)
except:
print "Warning: Please upgrade to a PyQT version >= 4.4"
def AutoDetectLangFromContent(self):
all_langs = []
for sub in self._subs:
if sub:
lang = sub.getLanguage()
if lang == None:
lang = Subtitle.AutoDetectLang(sub.getFilePath())
sub.setLanguage(lang)
all_langs.append(lang)
#FIXME: Clean this code here and put it in a shared script for also CLI to use
max = 0
max_lang = ""
for lang in all_langs:
if all_langs.count(lang) > max:
max = all_langs.count(lang)
max_lang = lang
xxx_lang = languages.name2xxx(max_lang)
log.debug("Majoritary Language Autodetected by content = " + str(xxx_lang))
if xxx_lang:
self._main.uploadLanguages.emit(SIGNAL('language_updated(QString,QString)'),xxx_lang, "content")
def __init__(self, parent, user_locale):
QtGui.QDialog.__init__(self)
self.ui = Ui_ChooseLanguageDialog()
self.ui.setupUi(self)
self._main = parent
settings = QSettings()
self.ui.languagesList.activated.connect(self.onOkButton)
self.ui.OKButton.clicked.connect(self.onOkButton)
for lang_locale in self._main.interface_langs:
languageName = Languages.locale2name(lang_locale)
if not languageName:
languageName = lang_locale
item = QListWidgetItem(languageName)
item.setData(Qt.UserRole, lang_locale)
self.ui.languagesList.addItem(item)
try:
if lang_locale == user_locale:
self.ui.languagesList.setCurrentItem(
item, QItemSelectionModel.ClearAndSelect)
except:
print("Warning: Please upgrade to a PyQT version >= 4.4")
def setUpClass(cls) -> None:
#Test database setup
cls.dbConnector = DatabaseConnector(
databaseFileName="testLanguageDB.db")
cls.dbConnector.createDatabase()
cls.dbConnector.openDatabaseConnection()
cls.dbConnector.executeSQL(
sql=
"CREATE TABLE Languages (ID INTEGER, Language TEXT, Bytes_of_Code INTEGER, PRIMARY KEY(ID))"
)
#Test data
cls.testData = {'testLang1': 7357, 'testLang2': 5}
cls.dataComparison = [(0, 'testLang1', 7357), (1, 'testLang2', 5)]
#Commit class setup
cls.lang = Languages(dbConnection=cls.dbConnector,
oauthToken="token",
repository="repo",
username="******",
url="url")
def AutoDetectLangFromContent(self):
all_langs = []
for sub in self._subs:
if sub:
lang = sub.getLanguage()
if lang == None:
lang = Subtitle.AutoDetectLang(sub.getFilePath())
sub.setLanguage(lang)
all_langs.append(lang)
# FIXME: Clean this code here and put it in a shared script for also CLI to use
max = 0
max_lang = ""
for lang in all_langs:
if all_langs.count(lang) > max:
max = all_langs.count(lang)
max_lang = lang
xxx_lang = languages.name2xxx(max_lang)
log.debug("Majoritary Language Autodetected by content = " +
str(xxx_lang))
if xxx_lang:
self._main.language_updated.emit(xxx_lang, "content")
import re
from flask import Blueprint, request, session, g, render_template, url_for, redirect
from pymongo import ASCENDING, DESCENDING
from pymongo.objectid import ObjectId
from pymongo.errors import InvalidId, PyMongoError
# Imports inside bombolone
from decorators import check_authentication, check_admin, get_hash_pages
from languages import Languages
from validators import CheckValue
MODULE_DIR = 'modules/pages'
pages = Blueprint('pages', __name__)
check = CheckValue() # Check Value class
languages_object = Languages()
class Pages(object):
""" This class allows to :
- get_page
- reset
- new
- remove
"""
page = {}
type_label = {}
len_of_label = 0
message = None # Error or succcess message
def setLanguageXXX(self, xxx):
self._languageXXX = xxx
self._languageXX = languages.xxx2xx(xxx)
self._languageName = languages.xxx2name(xxx)
def startDataCollection(self) -> None:
def _collectData(collector) -> int or bool:
data = collector.getData()
collector.insertData(dataset=data[0])
return collector.iterateNext(data[1])
def _scrapeData(collector) -> int or bool:
collector.insertData()
return 0
def _showProgression(collector, maxIterations: int) -> None:
for iteration in tqdm(range(0, abs(maxIterations) - 1), ):
_collectData(collector)
databaseConnection = self.checkForFile()
self.createFileTablesColumns(dbConnection=databaseConnection)
branchCollector = Branches(
dbConnection=self.dbConnector,
oauthToken=self.token,
repository=self.repository,
username=self.username,
url=
"https://api.github.com/repos/{}/{}/branches?per_page=100&page={}",
)
forksCollector = Forks(
dbConnection=self.dbConnector,
oauthToken=self.token,
repository=self.repository,
username=self.username,
url="https://api.github.com/repos/{}/{}/forks?per_page=100&page={}",
)
issuesCollector = Issues(
dbConnection=self.dbConnector,
oauthToken=self.token,
repository=self.repository,
username=self.username,
url=
"https://api.github.com/repos/{}/{}/issues?state=all&per_page=100&page={}",
)
languageCollector = Languages(
dbConnection=self.dbConnector,
oauthToken=self.token,
repository=self.repository,
username=self.username,
url=
"https://api.github.com/repos/{}/{}/languages?per_page=100&page={}",
)
repositoryCollector = Repository(
dbConnection=self.dbConnector,
oauthToken=self.token,
repository=self.repository,
username=self.username,
url="https://api.github.com/repos/{}/{}?per_page=100&page={}",
)
print("\nRepository Languages")
languagePages = _collectData(languageCollector) # One request only
_showProgression(languageCollector, languagePages)
print("\nRepository Information")
repositoryPages = _collectData(repositoryCollector) # One request only
_showProgression(repositoryCollector, repositoryPages)
print("\nRepository Branches")
branchPages = _collectData(branchCollector) # Estimated < 10 requests
_showProgression(branchCollector, branchPages)
print("\nRepository Forks")
forkPages = _collectData(forksCollector) # Estimated < 10 requests
_showProgression(forksCollector, forkPages)
print("\nRepository Issues")
issuePages = _collectData(issuesCollector) # Estimated < 20 requests
_showProgression(issuesCollector, issuePages)
commitsID = 0
branchList = self.dbConnector.selectColumn(table="Branches",
column="Name")
for branch in branchList:
print("\nRepository Commits from Branch {}".format(branch[0]))
commitsCollector = Commits(
dbConnection=self.dbConnector,
id=commitsID,
oauthToken=self.token,
repository=self.repository,
sha=branch[0],
username=self.username,
url=
"https://api.github.com/repos/{}/{}/commits?per_page=100&page={}&sha={}",
)
commitPages = _collectData(
commitsCollector) # Estimated to have the most requests
_showProgression(commitsCollector, commitPages)
commitsID = commitsCollector.exportID()
# TODO: Implement a loading bar for the Files module
# TODO: Reduce complexity where possible in the Files module
# Creates a combined list of every commit paired with its corresponding branch
branchList = self.dbConnector.selectColumn(table="Commits",
column="Branch")
commitSHAList = self.dbConnector.selectColumn(table="Commits",
column="Commit_SHA")
# https://www.geeksforgeeks.org/python-merge-two-lists-into-list-of-tuples/
mergedList = tuple(zip(branchList, commitSHAList))
filesID = 0
for pair in mergedList:
branch = pair[0][0]
commit = pair[1][0]
print("\nRepository Files from Branch {} from Commit {}".format(
branch, commit))
filesCollector = Files(
commitSHA=commit,
branch=branch,
dbConnection=self.dbConnector,
id=filesID,
repository=self.repository,
username=self.username,
url="https://github.com/{}/{}/commit/{}",
)
_scrapeData(filesCollector)
filesID = filesCollector.exportID()
def setLanguageName(self,language):
self._languageName = language
self._languageXXX = languages.name2xxx(language)
self._languageXX = languages.name2xx(language)
def setLanguageXXX(self,xxx):
self._languageXXX = xxx
self._languageXX = languages.xxx2xx(xxx)
self._languageName= languages.xxx2name(xxx)
def setLanguageName(self, language):
self._languageName = language
self._languageXXX = languages.name2xxx(language)
self._languageXX = languages.name2xx(language)
