def validate_against_schema(uploaded_file):
"""
Check if XML files that are part of a single upload validate
against the TrendMiner XML schema.
This function uses the `xmllint` tool to check if a given XML file
conforms to the TrendMiner XML schema. The schema is defined in
`<project-dir>/trendminer.xsd`. For any file that validates
against the schema, the xmllint tool returns 0.
"""
file_type = get_file_ext(uploaded_file.name)
if file_type == ".zip" and uploaded_file.folder:
for file_name in listdir(get_tmp_path(uploaded_file.folder)):
if file_name.endswith(".xml") and not file_name == "om.xml":
command = shlex.split(
'xmllint --noout --schema "{0}" "{1}"'.format(
SCHEMA_PATH, get_tmp_path(uploaded_file.folder, file_name)
)
)
subproc = subprocess.Popen(command)
returncode = subproc.wait()
if not returncode == 0:
raise ValidationError(UploadFormErrors.FILES_SCHEMA_CONFORMITY)
elif file_type == ".xml":
command = shlex.split(
'xmllint --noout --schema "{0}" "{1}"'.format(SCHEMA_PATH, get_tmp_path(uploaded_file.name))
)
subproc = subprocess.Popen(command)
if not subproc.wait() == 0:
raise ValidationError(UploadFormErrors.XML_SCHEMA_CONFORMITY)
def __open_test_file(self, file_path):
"""
Open file located at `file_path` using the appropriate
permissions and return it.
"""
flag = 'rb' if not get_file_ext(file_path) == '.xml' else 'r'
return open(file_path, flag)
def validate_xml_well_formedness(uploaded_file):
"""
Check if XML files that are part of a single upload are
well-formed.
This function uses the `xmlwf` tool to determine if a given XML
file is well-formed. The tool does not use standard return codes
for representing the outcome of the check. Instead, if a file is
well-formed, it simply outputs nothing. If it's not, xmlwf writes
a description of the problem to standard output.
"""
file_type = get_file_ext(uploaded_file.name)
if file_type == ".zip" and uploaded_file.folder:
for file_name in listdir(get_tmp_path(uploaded_file.folder)):
if file_name.endswith(".xml") and not file_name == "om.xml":
command = shlex.split('xmlwf "{}"'.format(get_tmp_path(uploaded_file.folder, file_name)))
subproc = subprocess.Popen(command, stdout=subprocess.PIPE)
error_msg = subproc.stdout.read()
if error_msg:
raise ValidationError(UploadFormErrors.FILES_WELLFORMEDNESS)
elif file_type == ".xml":
command = shlex.split('xmlwf "{}"'.format(get_tmp_path(uploaded_file.name)))
subproc = subprocess.Popen(command, stdout=subprocess.PIPE)
error_msg = subproc.stdout.read()
if error_msg:
raise ValidationError(UploadFormErrors.XML_WELLFORMEDNESS)
def sr_from_path(model, lr_path, save_dir):
ext = utils.get_file_ext(lr_path)
lr_image = get_image(lr_path, ext)
sr_image = model.predict(lr_image, steps=1)
sr_image = sr_image.clip(-1, 1)
lr_filename = utils.get_filename(lr_path)
sr_filename = lr_filename
save_image(sr_image, save_dir, sr_filename, ext)
def _analyse(data):
"""
Run TrendMiner on user uploaded data.
"""
file_type = get_file_ext(data.name)
if file_type == '.zip':
command = 'perl -I {0} {1}'.format(
PERL_PATH, path.join(PERL_PATH, 'om-xml.pl'))
subprocess.call(
command, cwd=get_tmp_path(data.folder), shell=True)
entities = parse_results(data.folder)
elif file_type == '.xml':
entities = []
return entities
def make_post_lite(row):
if row:
p = PostLite(
row['post_id'],
row['post_board_id'],
row['user'],
row['post'],
row['img_filename'],
row['img_uid'],
get_file_ext(row['img_filename']),
make_date(row['date']),
row['ip'],
)
return p
return None
def validate_mime_type(uploaded_file):
"""
Check MIME type of uploaded file and make sure it corresponds to
the file's extension.
This function uses the UNIX `file` command with the `--mime-type`
option to obtain the MIME type of the uploaded file. It then
checks to see if the MIME type corresponds to one of the types
appropriate for the file's extension.
"""
subproc = subprocess.Popen(
"file --mime-type {}".format(get_tmp_path(uploaded_file.name)), shell=True, stdout=subprocess.PIPE
)
mime_type = subproc.stdout.read().strip().split(": ")[-1]
file_extension = get_file_ext(uploaded_file.name)
if file_extension == ".zip" and not mime_type in ZIP_MIME_TYPES:
raise ValidationError(UploadFormErrors.MIME_TYPE.format(".zip", mime_type))
elif file_extension == ".xml" and not mime_type in XML_MIME_TYPES:
raise ValidationError(UploadFormErrors.MIME_TYPE.format(".xml", mime_type))
def make_replies(rows):
replies = None
if rows:
replies = []
for r in rows:
_r = Reply(
int(r['reply_id']),
int(r['reply_post_id']),
None
if not r['parent_reply_id'] else int(r['parent_reply_id']),
[],
[],
r['user'],
'' if r['reply'] is None else r['reply'],
r['img_filename'],
r['img_uid'],
get_file_ext(r['img_filename']),
make_date(r['date']),
r['ip'],
)
replies.append(_r)
return replies
def test_get_file_ext(self):
"""
Check if `get_file_ext` function correctly returns the
extension part of a given file name.
"""
self.assertEquals(get_file_ext(self.file_name), '.txt')
def validate_extension(uploaded_file):
"""
Check if extension of uploaded file is listed in `ACCEPTED_FILE_TYPES`.
"""
if not get_file_ext(uploaded_file.name.lower()) in ACCEPTED_FILE_TYPES:
raise ValidationError(UploadFormErrors.EXTENSION)
