def process_dump(self, dump_file):
"""
Generator that iterates through xml elements of given file and
extracts values from the ones that matches entity_path.
:param dump_file: File object of the dump.
:return: Triples of entity id, property id and external values
"""
node_path = []
for event, element in etree.iterparse(dump_file, events=("start", "end")):
if event == "start":
node_path.append(element.tag)
if event == "end":
if "/".join(node_path) == self.entities_path:
for external_value in self.process_entity(element):
if external_value is not None:
yield external_value
self.clean_up_references(element)
elif not "/".join(node_path).startswith(self.entities_path):
self.clean_up_references(element)
del node_path[-1]
if not self.is_quiet:
message = "Processing database dump...{0}"
consoleutils.print_progress(message, dump_file.tell())
# Write new line to console to overwrite progress
if not self.is_quiet:
print
def download_file(url, destination_file,
is_quiet=False, progress_message="Downloading...{0}"):
"""
Downloads file specified by url to given file object.
:param url: Url of the file that should be downloaded
:param destination_file: File, in which downloaded file should be written.
:param is_quiet: If set to True, console output will be suppressed.
:param progress_message: Message that shown on progress updates.
:return: Size of downloaded file.
"""
try:
response = urllib2.urlopen(url, timeout=DOWNLOAD_TIMEOUT)
except urllib2.URLError as exception:
raise DownloadError.DownloadError(message=exception.reason)
status_code = response.getcode()
if status_code == 200:
downloaded_bytes = 0
total_bytes = get_content_length(response)
while True:
download_buffer = response.read(DOWNLOAD_BUFFER_SIZE)
if not download_buffer:
break
downloaded_bytes += len(download_buffer)
destination_file.write(download_buffer)
if not is_quiet:
consoleutils.print_progress(progress_message,
downloaded_bytes, total_bytes)
destination_file.flush()
destination_file.seek(0)
# Write new line to console to overwrite progress
if not is_quiet:
print
return downloaded_bytes
else:
message = "HTTP response returned status code " + str(status_code)
raise DownloadError.DownloadError(status_code, message)
def test_print_progress(message, current_bytes, total_bytes, expected_output, capsys):
consoleutils.print_progress(message, current_bytes, total_bytes)
out, err = capsys.readouterr()
assert "\r\x1b[K" + expected_output == str(out)
