def find_range_components(meta):
file_size = 0
seek_pos = 0
hdr_range = meta.getheaders("Content-Range")
if len(hdr_range) > 0:
range_comp1 = hdr_range[0].split('/')
if len(range_comp1) > 1:
range_comp2 = range_comp1[0].split(
' ') #split away the "bytes" prefix
if len(range_comp2) == 0:
raise FatalException(
12, 'Malformed Content-Range response header: "{0}".' %
hdr_range)
range_comp3 = range_comp2[1].split('-')
seek_pos = int(range_comp3[0])
if range_comp1[1] != '*': #'*' == unknown length
file_size = int(range_comp1[1])
if file_size == 0:
#Try the old-fashioned way
hdrLen = meta.getheaders("Content-Length")
if len(hdrLen) == 0:
raise FatalException(
12,
"Response header doesn't contain Content-Length. Chunked Transfer-Encoding is not supported for now."
)
file_size = int(hdrLen[0])
return (file_size, seek_pos)
def _check_reference(cls, reference: Script):
if not reference or not reference.lang_ref:
raise FatalException('Reference not specified', [
'No reference was specified in course yaml file.',
'Do not know how to generate input.',
'Please add reference field to the course yaml file:'
'',
' reference:',
' name: main.py',
' lang: PY-367',
])
return True
def run_tests(self, api_tests: List[API_Test]):
try:
if not isinstance(api_tests, list):
raise FatalException(
'api_tests parameter must be a list of api_tests')
initial_test(silent=True)
print('----------------------------------------')
print(f'Running {len(api_tests)} tests')
print('----------------------------------------')
success_count = 0
for api_test in api_tests: # type: API_Test
try:
api_test.run_test()
except FailureException as e:
print(
f'Test failed {api_test}{" - " + api_test.description if api_test.description != "" else ""} - {e}\n'
)
else:
if self.log_success:
print(f'Test passed {api_test}\n')
success_count += 1
except FatalException as e:
print('----------------------------------------')
print(e)
except KeyboardInterrupt:
print('----------------------------------------')
print('Stopped')
except Exception as e:
print('----------------------------------------')
print(f'Unhandled and Unknown exception\n{e.__class__}\n{e.args}')
raise e
else:
print('----------------------------------------')
print(
f'Done, passed {success_count}/{len(api_tests)} tests successfully'
)
def _compile_cmd(self):
if self.request.type is ProcessRequestType.SOLVE:
pipeline = self.request.lang.compile
name = 'main.%s' % self.request.lang.extension
elif self.request.type in (ProcessRequestType.GENERATE_INPUT,
ProcessRequestType.GENERATE_OUTPUT):
if self.request.problem.reference:
pipeline = self.request.problem.reference.lang_ref.compile
name = self.request.problem.reference.name
else:
pipeline, name = None, None
else:
logger.error('Invalid action type')
raise FatalException('Invalid action type {}', self.request.type)
if not pipeline:
return None
return _configure_cmd(pipeline, name)
def error(cls, message, details=None):
return cls.emit(
'fatal-error',
dict(event='fatal-error', error=FatalException(message, details)))
def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func = None):
request = urllib2.Request(link)
if os.path.exists(destination) and not os.path.isfile(destination):
#Directory specified as target? Must be a mistake. Bail out, don't assume anything.
err = 'Download target {0} is a directory.'.format(destination)
raise FatalException(1, err)
(dest_path, file_name) = os.path.split(destination)
temp_dest = destination + ".tmpdownload"
partial_size = 0
if os.path.exists(temp_dest):
#Support for resuming downloads, in case the process is killed while downloading a file
# set resume range
# See http://stackoverflow.com/questions/6963283/python-urllib2-resume-download-doesnt-work-when-network-reconnects
partial_size = os.stat(temp_dest).st_size
if partial_size > chunk_size:
#Re-download the last chunk, to minimize the possibilities of file corruption
resume_pos = partial_size - chunk_size
request.add_header("Range", "bytes=%s-" % resume_pos)
else:
#Make sure the full dir structure is in place, otherwise file open will fail
if not os.path.exists(dest_path):
os.makedirs(dest_path)
response = urllib2.urlopen(request)
(file_size, seek_pos) = find_range_components(response.info())
print_info_msg("Downloading to: %s Bytes: %s" % (destination, file_size))
if partial_size < file_size:
if seek_pos == 0:
#New file, create it
open_mode = 'wb'
else:
#Resuming download of an existing file
open_mode = 'rb+' #rb+ doesn't create the file, using wb to create it
f = open(temp_dest, open_mode)
try:
#Resume the download from where it left off
if seek_pos > 0:
f.seek(seek_pos)
file_size_dl = seek_pos
while True:
buffer = response.read(chunk_size)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
if progress_func is not None:
progress_func(file_name, file_size_dl, chunk_size, file_size)
finally:
f.close()
sys.stdout.write('\n')
sys.stdout.flush()
print_info_msg("Finished downloading {0} to {1}".format(link, destination))
downloaded_size = os.stat(temp_dest).st_size
if downloaded_size != file_size:
err = 'Size of downloaded file {0} is {1} bytes, it is probably damaged or incomplete'.format(destination, downloaded_size)
raise FatalException(1, err)
# when download is complete -> mv temp_dest destination
if os.path.exists(destination):
#Windows behavior: rename fails if the destination file exists
os.unlink(destination)
os.rename(temp_dest, destination)
def __getitem__(self, id):
for course in self:
if course.id == id:
return course
raise FatalException('Could not find course %s' % id)
