def run():
from mod_home.models import CCExtractorVersion, GeneralData
from mod_regression.models import Category, RegressionTest, InputType, OutputType, RegressionTestOutput
from mod_sample.models import Sample
from mod_upload.models import Upload
from mod_auth.models import User
from database import create_session
db = create_session(sys.argv[1])
entries = []
categories = [
Category('Broken', 'Samples that are broken'),
Category('DVB', 'Samples that contain DVB subtitles'),
Category('DVD', 'Samples that contain DVD subtitles'),
Category('MP4', 'Samples that are stored in the MP4 format'),
Category('General', 'General regression samples')
]
entries.extend(categories)
samples = [
Sample('sample1', 'ts', 'sample1'),
Sample('sample2', 'ts', 'sample2')
]
entries.extend(samples)
cc_version = CCExtractorVersion(
'0.84', '2016-12-16T00:00:00Z',
'77da2dc873cc25dbf606a3b04172aa9fb1370f32')
entries.append(cc_version)
regression_tests = [
RegressionTest(1, '-autoprogram -out=ttxt -latin1', InputType.file,
OutputType.file, 3, 10),
RegressionTest(2, '-autoprogram -out=ttxt -latin1 -ucla',
InputType.file, OutputType.file, 1, 10)
]
entries.extend(regression_tests)
gen_data = GeneralData('last_commit',
'71dffd6eb30c1f4b5cf800307de845072ce33262')
entries.append(gen_data)
regression_test_output = [
RegressionTestOutput(1, "test1", "srt", "test1.srt"),
RegressionTestOutput(2, "test2", "srt", "test2.srt")
]
entries.extend(regression_test_output)
for entry in entries:
try:
db.add(entry)
db.commit()
except IntegrityError:
print("Entry already exists!", entry, flush=True)
db.rollback()
def run():
from mod_home.models import CCExtractorVersion, GeneralData
from mod_regression.models import Category, RegressionTest, InputType, \
OutputType
from mod_sample.models import Sample
from mod_upload.models import Upload
from mod_auth.models import User
from database import create_session
db = create_session(sys.argv[1])
categories = [
Category('Broken', 'Samples that are broken'),
Category('DVB', 'Samples that contain DVB subtitles'),
Category('DVD', 'Samples that contain DVD subtitles'),
Category('MP4', 'Samples that are stored in the MP4 format'),
Category('General', 'General regression samples')
]
db.add_all(categories)
db.commit()
samples = [
Sample('sample1', 'ts', 'sample1'),
Sample('sample2', 'ts', 'sample2')
]
db.add_all(samples)
db.commit()
cc_version = CCExtractorVersion(
'0.84', '2016-12-16', '77da2dc873cc25dbf606a3b04172aa9fb1370f32'
)
db.add(cc_version)
db.commit()
regression_tests = [
RegressionTest(
1, '-autoprogram -out=ttxt -latin1', InputType.file,
OutputType.file, 3, 10),
RegressionTest(
2, '-autoprogram -out=ttxt -latin1 -ucla', InputType.file,
OutputType.file, 1, 10)
]
db.add_all(regression_tests)
db.commit()
gen_data = GeneralData(
'last_commit', '71dffd6eb30c1f4b5cf800307de845072ce33262')
db.add(gen_data)
db.commit()
def process_id(upload_id):
from run import config
# Fetch upload id
queued_sample = QueuedSample.query.filter(
QueuedSample.id == upload_id).first()
if queued_sample is not None:
if queued_sample.user_id == g.user.id:
# Allowed to process
versions = CCExtractorVersion.query.all()
form = FinishQueuedSampleForm(request.form)
form.version.choices = [(v.id, v.version) for v in versions]
if form.validate_on_submit():
# Store in DB
db_committed = False
temp_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''),
'QueuedFiles', queued_sample.filename)
final_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''),
'TestFiles', queued_sample.filename)
try:
extension = queued_sample.extension[1:] if len(
queued_sample.extension) > 0 else ""
sample = Sample(queued_sample.sha, extension,
queued_sample.original_name)
g.db.add(sample)
g.db.flush([sample])
uploaded = Upload(g.user.id, sample.id, form.version.data,
Platform.from_string(form.platform.data),
form.parameters.data, form.notes.data)
g.db.add(uploaded)
g.db.delete(queued_sample)
g.db.commit()
db_committed = True
except:
traceback.print_exc()
g.db.rollback()
# Move file
if db_committed:
os.rename(temp_path, final_path)
return redirect(
url_for('sample.sample_by_id', sample_id=sample.id))
return {'form': form, 'queued_sample': queued_sample}
# Raise error
raise QueuedSampleNotFoundException()
def process_id(upload_id):
"""
Process the sample that is uploaded to the platform.
:param upload_id: The identity of uploaded file that will be processed
:type upload_id: str
:return: Process progress in form and queue the sample
:rtype: str
"""
from run import config, log
queued_sample = QueuedSample.query.filter(
QueuedSample.id == upload_id).first()
if queued_sample is not None:
if queued_sample.user_id == g.user.id:
versions = CCExtractorVersion.query.all()
form = FinishQueuedSampleForm(request.form)
form.version.choices = [(v.id, v.version) for v in versions]
if form.validate_on_submit():
db_committed = False
repo_folder = config.get('SAMPLE_REPOSITORY', '')
temp_path = os.path.join(repo_folder, 'QueuedFiles',
queued_sample.filename)
final_path = os.path.join(repo_folder, 'TestFiles',
queued_sample.filename)
try:
extension = queued_sample.extension[1:] if len(
queued_sample.extension) > 0 else ""
sample = Sample(queued_sample.sha, extension,
queued_sample.original_name)
g.db.add(sample)
g.db.flush([sample])
uploaded = Upload(g.user.id, sample.id, form.version.data,
Platform.from_string(form.platform.data),
form.parameters.data, form.notes.data)
g.db.add(uploaded)
g.db.delete(queued_sample)
g.db.commit()
db_committed = True
except Exception:
traceback.print_exc()
g.db.rollback()
if db_committed:
if form.report.data == 'y':
data = ""
try:
kvm_name = config.get('KVM_LINUX_NAME', '')
repo = Repo(
os.path.join(repo_folder, 'vm_data', kvm_name,
'unsafe-ccextractor'))
data = repo.git.show(
f"{repo.heads.master}:.github/ISSUE_TEMPLATE.md"
)
except InvalidGitRepositoryError:
log.critical(
"Could not open CCExtractor's repository")
version = CCExtractorVersion.query.filter(
CCExtractorVersion.id ==
form.version.data).first()
data = data.replace("**X.X**", version.version)
data = data.replace("[ ] I have read",
"[X] I have read")
data = data.replace("[ ] I have checked",
"[X] I have checked")
data = data.replace("[ ] I have used",
"[X] I have used")
data = data.replace(
"[ ] I am an active contributor to CCExtractor.",
"[X] I used the platform to submit this issue!")
data = data.replace("`-autoprogram`",
f"`{form.parameters.data}`")
platform = form.platform.data.title()
data = data.replace('[ ] ' + platform,
'[X] ' + platform)
# Remove everything starting from the video links
data = data[:data.find('**Video links**')]
# Append our own content here
sample_link = url_for('sample.sample_by_id',
sample_id=sample.id,
_external=True)
data += f"""
**Sample**
[Sample {sample.id}]({sample_link}) uploaded on the Sample Platform.
**Extra information**
*Notes:*
{form.notes.data}
*Description:*
{form.IssueBody.data}"""
issue_title = f"[BUG] {form.IssueTitle.data}"
issue_data = make_github_issue(
issue_title, data, ['bug', f'sample_{sample.id}'])
if issue_data != 'ERROR':
issue_id = issue_data['number']
issue_title = issue_data['title']
issue_user = issue_data['user']['login']
issue_date = issue_data['created_at']
issue_status = issue_data['state']
issue = Issue(sample.id, issue_id, issue_date,
issue_title, issue_user,
issue_status)
g.db.add(issue)
g.db.commit()
else:
flash(
"Could not submit an issue on GitHub (did you revoke permissions for the platform?)."
" Please submit it manually.")
os.rename(temp_path, final_path)
return redirect(
url_for('sample.sample_by_id', sample_id=sample.id))
return {'form': form, 'queued_sample': queued_sample}
# Raise error
raise QueuedSampleNotFoundException()
def setUp(self):
self.app.preprocess_request()
g.db = create_session(self.app.config['DATABASE_URI'],
drop_tables=True)
# enable Foreign keys for unit tests
g.db.execute('pragma foreign_keys=on')
general_data = [
GeneralData('last_commit',
"1978060bf7d2edd119736ba3ba88341f3bec3323"),
GeneralData(f'fetch_commit_{TestPlatform.linux.value}',
"1978060bf7d2edd119736ba3ba88341f3bec3323"),
GeneralData(f'fetch_commit_{TestPlatform.windows.value}',
"1978060bf7d2edd119736ba3ba88341f3bec3323")
]
g.db.add_all(general_data)
self.ccextractor_version = CCExtractorVersion(
"1.2.3", "2013-02-27T19:35:32Z",
"1978060bf7d2edd119736ba3ba88341f3bec3323")
g.db.add(self.ccextractor_version)
fork = Fork(
f"https://github.com/{g.github['repository_owner']}/{g.github['repository']}.git"
)
g.db.add(fork)
g.db.commit()
dummy_user = User(signup_information['existing_user_name'],
signup_information['existing_user_role'],
signup_information['existing_user_email'],
signup_information['existing_user_pwd'])
g.db.add(dummy_user)
g.db.commit()
test = [
Test(TestPlatform.linux, TestType.pull_request, 1, "master",
"1978060bf7d2edd119736ba3ba88341f3bec3323", 1),
Test(TestPlatform.linux, TestType.pull_request, 1, "master",
"abcdefgh", 1)
]
g.db.add_all(test)
g.db.commit()
categories = [
Category("Broken", "Samples that are broken"),
Category("DVB", "Samples that contain DVB subtitles"),
Category("DVD", "Samples that contain DVD subtitles"),
Category("MP4", "Samples that are stored in the MP4 format"),
Category("General", "General regression samples")
]
g.db.add_all(categories)
g.db.commit()
samples = [
Sample("sample1", "ts", "sample1"),
Sample("sample2", "ts", "sample2")
]
g.db.add_all(samples)
g.db.commit()
upload = [
Upload(1, 1, 1, Platform.windows),
Upload(1, 2, 1, Platform.linux)
]
g.db.add_all(upload)
g.db.commit()
regression_tests = [
RegressionTest(1, "-autoprogram -out=ttxt -latin1 -2",
InputType.file, OutputType.file, 3, 10),
RegressionTest(2, "-autoprogram -out=ttxt -latin1 -ucla",
InputType.file, OutputType.file, 1, 10)
]
g.db.add_all(regression_tests)
g.db.commit()
categories[0].regression_tests.append(regression_tests[0])
categories[2].regression_tests.append(regression_tests[1])
regression_test_outputs = [
RegressionTestOutput(1, "sample_out1", ".srt", ""),
RegressionTestOutput(2, "sample_out2", ".srt", "")
]
g.db.add_all(regression_test_outputs)
g.db.commit()
rtof = RegressionTestOutputFiles("bluedabadee", 2)
g.db.add(rtof)
g.db.commit()
test_result_progress = [
TestProgress(1, TestStatus.preparation, "Test 1 preparation"),
TestProgress(1, TestStatus.building, "Test 1 building"),
TestProgress(1, TestStatus.testing, "Test 1 testing"),
TestProgress(1, TestStatus.completed, "Test 1 completed"),
TestProgress(2, TestStatus.preparation, "Test 2 preparation"),
TestProgress(2, TestStatus.building, "Test 2 building"),
TestProgress(2, TestStatus.testing, "Test 2 testing"),
TestProgress(2, TestStatus.completed, "Test 2 completed")
]
g.db.add_all(test_result_progress)
g.db.commit()
test_results = [
TestResult(1, 1, 200, 0, 0),
TestResult(1, 2, 601, 0, 0),
TestResult(2, 1, 200, 200, 0),
TestResult(2, 2, 601, 0, 0)
]
g.db.add_all(test_results)
g.db.commit()
test_result_files = [
TestResultFile(1, 1, 1, "sample_out1"),
TestResultFile(1, 2, 2, "sample_out2"),
TestResultFile(2, 1, 1, "sample_out1"),
TestResultFile(2, 2, 2, "sample_out2", "out2")
]
g.db.add_all(test_result_files)
g.db.commit()
forbidden_mime = ForbiddenMimeType("application/javascript")
forbidden_ext = [ForbiddenExtension("js"), ForbiddenExtension("com")]
g.db.add(forbidden_mime)
g.db.add_all(forbidden_ext)
g.db.commit()
def setUp(self):
self.app.preprocess_request()
g.db = create_session(self.app.config['DATABASE_URI'],
drop_tables=True)
g.db.execute('pragma foreign_keys=on') # Enable Foreign for unit tests
commit_name_linux = 'fetch_commit_' + TestPlatform.linux.value
commit_name_windows = 'fetch_commit_' + TestPlatform.windows.value
general_data = [
GeneralData('last_commit',
'1978060bf7d2edd119736ba3ba88341f3bec3323'),
GeneralData(commit_name_linux,
'1978060bf7d2edd119736ba3ba88341f3bec3323'),
GeneralData(commit_name_windows,
'1978060bf7d2edd119736ba3ba88341f3bec3323')
]
g.db.add_all(general_data)
self.ccextractor_version = CCExtractorVersion(
'1.2.3', '2013-02-27T19:35:32Z',
'1978060bf7d2edd119736ba3ba88341f3bec3323')
g.db.add(self.ccextractor_version)
fork_url = ('https://github.com/{user}/{repo}.git').format(
user=g.github['repository_owner'], repo=g.github['repository'])
fork = Fork(fork_url)
g.db.add(fork)
g.db.commit()
dummy_user = User(signup_information['existing_user_name'],
signup_information['existing_user_role'],
signup_information['existing_user_email'],
signup_information['existing_user_pwd'])
g.db.add(dummy_user)
g.db.commit()
test = [
Test(TestPlatform.linux, TestType.pull_request, 1, 'master',
'1978060bf7d2edd119736ba3ba88341f3bec3323', 1),
Test(TestPlatform.linux, TestType.pull_request, 1, 'master',
'abcdefgh', 1)
]
g.db.add_all(test)
g.db.commit()
categories = [
Category('Broken', 'Samples that are broken'),
Category('DVB', 'Samples that contain DVB subtitles'),
Category('DVD', 'Samples that contain DVD subtitles'),
Category('MP4', 'Samples that are stored in the MP4 format'),
Category('General', 'General regression samples')
]
g.db.add_all(categories)
g.db.commit()
samples = [
Sample('sample1', 'ts', 'sample1'),
Sample('sample2', 'ts', 'sample2')
]
g.db.add_all(samples)
g.db.commit()
upload = [
Upload(1, 1, 1, Platform.windows),
Upload(1, 2, 1, Platform.linux)
]
g.db.add_all(upload)
g.db.commit()
regression_tests = [
RegressionTest(1, '-autoprogram -out=ttxt -latin1 -2',
InputType.file, OutputType.file, 3, 10),
RegressionTest(2, '-autoprogram -out=ttxt -latin1 -ucla',
InputType.file, OutputType.file, 1, 10)
]
g.db.add_all(regression_tests)
g.db.commit()
categories[0].regression_tests.append(regression_tests[0])
categories[2].regression_tests.append(regression_tests[1])
regression_test_outputs = [
RegressionTestOutput(1, 'sample_out1', '.srt', ''),
RegressionTestOutput(2, 'sample_out2', '.srt', '')
]
g.db.add_all(regression_test_outputs)
g.db.commit()
test_result_progress = [
TestProgress(1, TestStatus.preparation, "Test 1 preperation"),
TestProgress(1, TestStatus.building, "Test 1 building"),
TestProgress(1, TestStatus.testing, "Test 1 testing"),
TestProgress(1, TestStatus.completed, "Test 1 completed"),
TestProgress(2, TestStatus.preparation, "Test 2 preperation"),
TestProgress(2, TestStatus.building, "Test 2 building"),
TestProgress(2, TestStatus.testing, "Test 2 testing"),
TestProgress(2, TestStatus.completed, "Test 2 completed")
]
g.db.add_all(test_result_progress)
g.db.commit()
test_results = [
TestResult(1, 1, 200, 0, 0),
TestResult(1, 2, 601, 0, 0),
TestResult(2, 1, 200, 200, 0),
TestResult(2, 2, 601, 0, 0)
]
g.db.add_all(test_results)
g.db.commit()
test_result_files = [
TestResultFile(1, 1, 1, 'sample_out1'),
TestResultFile(1, 2, 2, 'sample_out2'),
TestResultFile(2, 1, 1, 'sample_out1'),
TestResultFile(2, 2, 2, 'sample_out2', 'out2')
]
g.db.add_all(test_result_files)
g.db.commit()
forbidden_mime = ForbiddenMimeType('application/javascript')
forbidden_ext = [ForbiddenExtension('js'), ForbiddenExtension('com')]
g.db.add(forbidden_mime)
g.db.add_all(forbidden_ext)
g.db.commit()