def init_download_file_pointers(self, bucket_id, file_id):
try:
# logger.warning("log_event_type": "debug")
logger.debug('"title": "File pointers"')
logger.debug('"description": "Resolving file pointers to download\
file with ID: "' + str(file_id) + "...")
# logger.warning(str({"log_event_type": "debug", "title": "File pointers",
# "description": "Resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
file_pointers = self.storj_engine.storj_client.file_pointers(
str(bucket_id), file_id)
self.emit(QtCore.SIGNAL("beginDownloadProccess"), file_pointers)
except storj.exception.StorjBridgeApiError as e:
# logger.warning("log_event_type": "error")
logger.debug('"title": "Bridge error"')
logger.debug('"description": "Error while resolving file pointers \
to download file with ID: "' + str(file_id) + "...")
# logger.warning(str({"log_event_type": "error", "title": "Bridge error",
# "description": "Error while resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
self.emit(QtCore.SIGNAL("showStorjBridgeException"),
str(e)) # emit Storj Bridge Exception
except Exception as e:
# logger.warning('"log_event_type": "error"')
logger.debug('"title": "Unhandled error"'),
logger.debug('"description": "Unhandled error while resolving file\
pointers to download file with ID: "' + str(file_id) +
"...")
# logger.warning(str({"log_event_type": "error", "title": "Unhandled error",
# "description": "Unhandled error while resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
self.emit(QtCore.SIGNAL("showUnhandledException"),
str(e)) # emit unhandled Exception
logger.error(e)
def open_mirror_details_window(self, mirror_state):
# self.established_mirrors_tree_view = self.file_mirrors_list_ui.established_mirrors_tree
# daat = self.file_mirrors_list_ui.established_mirrors_tree.selectedIndexes()
# model = self.file_mirrors_list_ui.established_mirrors_tree.model()
# data = []
# initialize variables
item = ""
index = ""
try:
if mirror_state == "established":
index = self.file_mirrors_list_ui.established_mirrors_tree.selectedIndexes()[3]
item = self.file_mirrors_list_ui.established_mirrors_tree.selectedIndexes()[3]
elif mirror_state == "available":
index = self.file_mirrors_list_ui.available_mirrors_tree.selectedIndexes()[3]
item = self.file_mirrors_list_ui.available_mirrors_tree.selectedIndexes()[3]
nodeid_to_send = item.model().itemFromIndex(index).text()
if nodeid_to_send != "":
self.node_details_window = NodeDetailsUI(self, nodeid_to_send)
self.node_details_window.show()
else:
QtGui.QMessageBox.about(self, "Warning", "Please select farmer node from list")
logger.warning("Unhandled error")
except:
QtGui.QMessageBox.about(self, "Warning", "Please select farmer node from list")
logger.error("Unhandled error")
def show_storj_bridge_exception(self, exception_content):
try:
j = json.loads(str(exception_content))
QtGui.QMessageBox.critical(self, "Bridge error", str(j["error"]))
except Exception as e:
QtGui.QMessageBox.critical(self, "Bridge error", str(exception_content))
logger.error(e)
def update_files_list(self):
self.tools = Tools()
model = QtGui.QStandardItemModel(1, 1) # initialize model for inserting to table
model.setHorizontalHeaderLabels(['File name', 'File size', 'Mimetype', 'File ID'])
self.current_bucket_index = self.file_manager_ui.bucket_select_combo_box.currentIndex()
self.current_selected_bucket_id = self.bucket_id_list[self.current_bucket_index]
i = 0
try:
for self.file_details in self.storj_engine.storj_client.bucket_files(str(self.current_selected_bucket_id)):
item = QtGui.QStandardItem(str(self.file_details["filename"]))
model.setItem(i, 0, item) # row, column, item (StandardItem)
file_size_str = self.tools.human_size(int(self.file_details["size"])) # get human readable file size
item = QtGui.QStandardItem(str(file_size_str))
model.setItem(i, 1, item) # row, column, item (QQtGui.StandardItem)
item = QtGui.QStandardItem(str(self.file_details["mimetype"]))
model.setItem(i, 2, item) # row, column, item (QStandardItem)
item = QtGui.QStandardItem(str(self.file_details["id"]))
model.setItem(i, 3, item) # row, column, item (QStandardItem)
i = i + 1
logger.info(self.file_details)
except sjexc.StorjBridgeApiError as e:
logger.error(e)
self.file_manager_ui.files_list_tableview.clearFocus()
self.file_manager_ui.files_list_tableview.setModel(model)
self.file_manager_ui.files_list_tableview.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
def create_download_connection(self, url, path_to_save, options_chain,
progress_bar, rowposition, shard_index):
local_filename = path_to_save
downloaded = False
farmer_tries = 0
# logger.warning('"log_event_type": "debug"')
logger.debug('"title": "Downloading"')
logger.debug('"description": "Downloading shard at index "' +
str(shard_index) + " from farmer: " + str(url))
# logger.warning(str({"log_event_type": "debug", "title": "Downloading",
# "description": "Downloading shard at index " + str(shard_index) + " from farmer: " + str(
# url)}))
tries_download_from_same_farmer = 0
while self.max_retries_download_from_same_farmer > tries_download_from_same_farmer:
tries_download_from_same_farmer += 1
farmer_tries += 1
try:
self.emit(QtCore.SIGNAL("updateDownloadTaskState"),
rowposition,
"Downloading...") # update shard downloading state
if options_chain["handle_progressbars"] != "1":
r = requests.get(url)
# requests.
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
else:
r = requests.get(url, stream=True)
f = open(local_filename, 'wb')
if options_chain["file_size_is_given"] == "1":
file_size = options_chain["shard_file_size"]
else:
file_size = int(r.headers['Content-Length'])
chunk = 1
num_bars = file_size / chunk
t1 = float(file_size) / float((32 * 1024))
logger.debug(t1)
if file_size <= (32 * 1024):
t1 = 1
i = 0
logger.debug(file_size)
logger.debug(str(t1) + "kotek")
for chunk in r.iter_content(32 * 1024):
i += 1
f.write(chunk)
logger.debug(str(i) + " " + str(t1))
logger.debug(round(float(i) / float(t1), 1))
logger.debug(str(int(round((100.0 * i) / t1))) + " %")
if int(round((100.0 * i) / t1)) > 100:
percent_downloaded = 100
else:
percent_downloaded = int(round((100.0 * i) / t1))
self.emit(
QtCore.SIGNAL("updateShardDownloadProgress"),
int(rowposition), percent_downloaded
) # update progress bar in upload queue table
self.shard_download_percent_list[
shard_index] = percent_downloaded
self.emit(
QtCore.SIGNAL("refreshOverallDownloadProgress"),
0.1) # update progress bar in upload queue table
logger.debug(str(rowposition) + "pozycja")
# progress_bar.setValue(percent_downloaded)
f.close()
downloaded = True
except Exception as e:
logger.error(e)
# logger.warning('"log_event_type": "warning"')
logger.debug('"title": "Unhandled error"')
logger.debug('"description": "Error occured while downloading\
shard at index "' + str(shard_index) +
". Retrying... (" + str(farmer_tries) + ")")
# logger.warning(str({"log_event_type": "warning", "title": "Unhandled error",
# "description": "Error occured while downloading shard at index " + str(
# shard_index) + ". Retrying... (" + str(farmer_tries) + ")"}))
self.emit(QtCore.SIGNAL("updateDownloadTaskState"),
rowposition, "First try failed. Retrying... (" +
str(farmer_tries) +
")") # update shard download state
continue
else:
downloaded = True
break
if not downloaded:
self.emit(QtCore.SIGNAL("retryWithNewDownloadPointer"),
options_chain["shard_index"]
) # retry download with new download pointer
else:
# logger.warning(str({"log_event_type": "success", "title": "Shard downloaded", "description": "Shard at index " + str(shard_index) + " downloaded successfully."}))
# logger.warning('"log_event_type": "success"')
logger.debug('"title": "Shard downloaded"')
logger.debug('"description": "Shard at index "' +
str(shard_index) + " downloaded successfully.")
self.emit(QtCore.SIGNAL("incrementShardsDownloadProgressCounters")
) # update already uploaded shards count
self.emit(QtCore.SIGNAL("updateDownloadTaskState"), rowposition,
"Downloaded!") # update shard download state
if int(self.all_shards_count) <= int(
self.shards_already_downloaded + 1):
self.emit(
QtCore.SIGNAL("finishDownload")
) # send signal to begin file shards joind and decryption after all shards are downloaded
return
def initialize_mirrors_tree(self):
# create model
# model = QtGui.QFileSystemModel()
# model.setRootPath(QtCore.QDir.currentPath())
self.file_mirrors_list_ui.loading_label_mirrors_established.setStyleSheet('color: red') # set loading color
self.file_mirrors_list_ui.loading_label_mirrors_available.setStyleSheet('color: red') # set loading color
self.mirror_tree_view_header = ['Shard Hash / Address', 'User agent', 'Last seed', 'Node ID']
######################### set the model for established mirrors ##################################
self.established_mirrors_model = QtGui.QStandardItemModel()
self.established_mirrors_model.setHorizontalHeaderLabels(self.mirror_tree_view_header)
self.established_mirrors_tree_view = self.file_mirrors_list_ui.established_mirrors_tree
self.established_mirrors_tree_view.setSelectionBehavior(QtGui.QAbstractItemView.SelectRows)
self.established_mirrors_tree_view.setModel(self.established_mirrors_model)
self.established_mirrors_tree_view.setUniformRowHeights(True)
self.file_mirrors_list_ui.available_mirrors_tree.setModel(self.established_mirrors_model)
divider = 0
group = 1
self.established_mirrors_count_for_file = 0
recent_shard_hash = ""
parent1 = QtGui.QStandardItem('')
try:
for file_mirror in self.storj_engine.storj_client.file_mirrors(str(self.bucketid), str(self.fileid)):
for mirror in file_mirror.established:
self.established_mirrors_count_for_file += 1
logger.info(file_mirror.established)
if mirror["shardHash"] != recent_shard_hash:
parent1 = QtGui.QStandardItem('Shard with hash {}'.format(mirror["shardHash"]))
divider = divider + 1
self.established_mirrors_model.appendRow(parent1)
child1 = QtGui.QStandardItem(str(mirror["contact"]["address"] + ":" + str(mirror["contact"]["port"])))
child2 = QtGui.QStandardItem(str(mirror["contact"]["userAgent"]))
child3 = QtGui.QStandardItem(str(mirror["contact"]["lastSeen"]))
child4 = QtGui.QStandardItem(str(mirror["contact"]["nodeID"]))
parent1.appendRow([child1, child2, child3, child4])
# span container columns
# self.established_mirrors_tree_view.setFirstColumnSpanned(1, self.established_mirrors_tree_view.rootIndex(), True)
recent_shard_hash = mirror["shardHash"]
self.file_mirrors_list_ui.loading_label_mirrors_established.setText("")
# dbQueryModel.itemData(treeView.selectedIndexes()[0])
################################### set the model for available mirrors #########################################
self.available_mirrors_model = QtGui.QStandardItemModel()
self.available_mirrors_model.setHorizontalHeaderLabels(self.mirror_tree_view_header)
self.available_mirrors_tree_view = self.file_mirrors_list_ui.available_mirrors_tree
self.available_mirrors_tree_view.setSelectionBehavior(QtGui.QAbstractItemView.SelectRows)
self.available_mirrors_tree_view.setModel(self.available_mirrors_model)
self.available_mirrors_tree_view.setUniformRowHeights(True)
self.file_mirrors_list_ui.available_mirrors_tree.setModel(self.available_mirrors_model)
divider = 0
self.available_mirrors_count_for_file = 0
recent_shard_hash_2 = ""
parent2 = QtGui.QStandardItem('')
for file_mirror in self.storj_engine.storj_client.file_mirrors(str(self.bucketid), str(self.fileid)):
for mirror_2 in file_mirror.available:
self.available_mirrors_count_for_file += 1
if mirror_2["shardHash"] != recent_shard_hash_2:
parent2 = QtGui.QStandardItem('Shard with hash {}'.format(mirror_2["shardHash"]))
divider = divider + 1
self.available_mirrors_model.appendRow(parent2)
child1 = QtGui.QStandardItem(str(mirror_2["contact"]["address"] + ":" + str(mirror_2["contact"]["port"])))
child2 = QtGui.QStandardItem(str(mirror_2["contact"]["userAgent"]))
child3 = QtGui.QStandardItem(str(mirror_2["contact"]["lastSeen"]))
child4 = QtGui.QStandardItem(str(mirror_2["contact"]["nodeID"]))
parent2.appendRow([child1, child2, child3, child4])
# span container columns
# self.established_mirrors_tree_view.setFirstColumnSpanned(1, self.established_mirrors_tree_view.rootIndex(), True)
recent_shard_hash_2 = mirror_2["shardHash"]
self.file_mirrors_list_ui.loading_label_mirrors_available.setText("")
self.file_mirrors_list_ui.established_mirrors_count.setText(
html_format_begin + str(self.established_mirrors_count_for_file) + html_format_end)
self.file_mirrors_list_ui.available_mirrors_count.setText(
html_format_begin + str(self.available_mirrors_count_for_file) + html_format_end)
except sjexc.StorjBridgeApiError as e:
self.emit(QtCore.SIGNAL("showStorjBridgeException"), str(e)) # emit Storj Bridge Exception
except Exception as e:
self.emit(QtCore.SIGNAL("showUnhandledException"), str(e)) # emit unhandled Exception
logger.error(e)
def shard_download(self, pointer, file_save_path, options_array):
# logger.warning(str({"log_event_type": "debug", "title": "Downloading", "description": "Beginning download proccess..."}))
# logger.warning('"log_event_type": "debug"')
#logger.debug('"title": "Downloading"')
logger.debug('Beginning download proccess...')
options_chain = {}
# self.storj_engine.storj_client.logger.info('file_pointers(%s, %s)', bucket_id, file_id)
file_name = os.path.split(file_save_path)[1]
##### End file download finish point #####
try:
# check ability to write files to selected directories
if self.tools.isWritable(os.path.split(file_save_path)[0]) != True:
raise IOError("13")
if self.tools.isWritable(self.tmp_path) != True:
raise IOError("13")
try:
if options_array["progressbars_enabled"] == "1":
options_chain["handle_progressbars"] = "1"
if options_array["file_size_is_given"] == "1":
options_chain["file_size_is_given"] = "1"
shards_count = int(options_array["shards_count"])
shard_size_array = []
shard_size_array.append(
int(options_array["file_size_shard_" +
str(options_array["shard_index"])]))
logger.debug(shard_size_array)
part = options_array["shard_index"]
self.tmp_path = options_array["tmp_path"]
self.emit(QtCore.SIGNAL("setCurrentState"),
"Starting download threads...")
self.emit(
QtCore.SIGNAL("setCurrentState"),
"Started download shard at index " + str(part) + "...")
options_chain["rowposition"] = part
self.shard_download_percent_list.append(0)
rowposition = self._add_shard_to_table(
pointer, 0, part) # Add a row to the table
logger.debug(pointer)
options_chain["shard_file_size"] = shard_size_array[0]
url = "http://" + pointer.get('farmer')['address'] + \
":" + \
str(pointer.get('farmer')['port']) + \
"/shards/" + pointer["hash"] + \
"?token=" + pointer["token"]
logger.debug(url)
if self.combine_tmpdir_name_with_token:
self.createNewDownloadThread(
url, self.tmp_path + "/" + str(pointer["token"]) +
"/" + str(file_name) + "-" + str(part), options_chain,
rowposition - 1, part)
else:
self.createNewDownloadThread(
url,
self.tmp_path + "/" + str(file_name) + "-" + str(part),
options_chain, rowposition - 1, part)
logger.debug(self.tmp_path + "/" + str(file_name) + "-" +
str(part))
part = part + 1
except Exception as e:
logger.error(e)
# logger.warning('"log_event_type": "warning"')
logger.debug('Unhandled error' + str(e))
# except Exception as e:
# self.emit(QtCore.SIGNAL("showStorjBridgeException"),
# "Unhandled error while resolving file pointers for download. " + str(
# e)) # emit unhandled Exception
except IOError as e:
print " perm error " + str(e)
if str(e) == str(13):
self.emit(QtCore.SIGNAL(
"showException"
), "Error while saving or reading file or temporary file. Probably this is caused by insufficient permisions. Please check if you have permissions to write or read from selected directories. "
) # emit Storj Bridge Exception
except Exception as e:
logger.error(e)
# logger.warning('"log_event_type": "warning"')
logger.debug('Unhandled error')
def download_begin(self, bucket_id, file_id):
self.overwrite_question_closed = False
self.validation = {}
self.all_shards_count = self.get_file_pointers_count(
bucket_id, file_id)
self.shards_already_downloaded = 0
self.destination_file_path = str(
self.ui_single_file_download.file_save_path.text()).decode('utf-8')
self.tmp_path = str(
self.ui_single_file_download.tmp_dir.text()).decode('utf-8')
if self.tmp_path == "":
if platform == "linux" or platform == "linux2":
# linux
self.tmp_path = "/tmp/"
elif platform == "darwin":
# OS X
self.tmp_path = "/tmp/"
elif platform == "win32":
# Windows
self.tmp_path = "C:\\Windows\\temp\\"
file_name = os.path.split(self.destination_file_path)[1]
if self.destination_file_path == "":
self.validation["file_path"] = False
self.emit(QtCore.SIGNAL("showDestinationPathNotSelectedMsg")
) # show error missing destination path
logger.error("missing destination file path")
else:
self.validation["file_path"] = True
if os.path.isfile(self.destination_file_path):
self.emit(QtCore.SIGNAL("askFileOverwrite"), str(file_name))
while not self.overwrite_question_closed:
pass
if self.overwrite_question_result == QtGui.QMessageBox.Yes:
self.validation["file_path"] = True
else:
self.validation["file_path"] = False
# emit signal to select new file path
self.emit(QtCore.SIGNAL("selectFileDestinationPath"))
if self.validation["file_path"]:
self.ui_single_file_download.start_download_bt.setStyleSheet(
("QPushButton:hover{\n"
" background-color: #8C8A87;\n"
" border-color: #8C8A87;\n"
"}\n"
"QPushButton:active {\n"
" background-color: #8C8A87;\n"
" border-color: #8C8A87;\n"
"}\n"
"QPushButton{\n"
" background-color: #8C8A87;\n"
" border: 1px solid #8C8A87;\n"
" color: #fff;\n"
" border-radius: 7px;\n"
"}"))
self.emit(QtCore.SIGNAL("updateShardCounters"))
try:
# logger.warning("log_event_type": "debug")
# logger.debug('"title": "File pointers"')
logger.debug('Resolving file pointers to download\
file with ID: ' + str(file_id) + "...")
# logger.warning(str({"log_event_type": "debug", "title": "File pointers",
# "description": "Resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
# get_file_pointers_count(self, bucket_id, file_id)
i = 0
# while i < self.all_shards_count:
while i < 4 and i < self.all_shards_count:
self.is_upload_active = True
tries_get_file_pointers = 0
while self.max_retries_get_file_pointers > tries_get_file_pointers:
tries_get_file_pointers += 1
time.sleep(1)
try:
options_array = {}
options_array["tmp_path"] = self.tmp_path
options_array["progressbars_enabled"] = "1"
options_array["file_size_is_given"] = "1"
options_array["shards_count"] = str(
self.all_shards_count)
shard_pointer = self.storj_engine.storj_client.file_pointers(
str(bucket_id),
file_id,
limit="1",
skip=str(i))
print str(shard_pointer) + "wskaznik"
# if shard_pointer[0]["parity"] == False:
# print "Shard parity error!"
# break
options_array["shard_index"] = shard_pointer[0][
"index"]
options_array["file_size_shard_" +
str(i)] = shard_pointer[0]["size"]
self.emit(
QtCore.SIGNAL("beginShardDownloadProccess"),
shard_pointer[0], self.destination_file_path,
options_array)
except stjex.StorjBridgeApiError as e:
logger.debug('"title": "Bridge error"')
logger.debug(
'"description": "Error while resolving file pointers \
to download file with ID: "'
+ str(file_id) + "...")
self.emit(
QtCore.SIGNAL("showStorjBridgeException"),
str(e)) # emit Storj Bridge Exception
continue
except Exception as e:
continue
else:
break
self.already_started_shard_downloads_count += 1
i += 1
except storj.exception.StorjBridgeApiError as e:
self.is_upload_active = False
# logger.warning("log_event_type": "error")
logger.debug('"title": "Bridge error"')
logger.debug(
'"description": "Error while resolving file pointers \
to download file with ID: "' +
str(file_id) + "...")
# logger.warning(str({"log_event_type": "error", "title": "Bridge error",
# "description": "Error while resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
self.emit(QtCore.SIGNAL("showStorjBridgeException"),
str(e)) # emit Storj Bridge Exception
# except Exception as e:
# logger.warning('"log_event_type": "error"')
# logger.debug('"title": "Unhandled error"'),
# logger.debug('"description": "Unhandled error while resolving file\
# pointers to download file with ID: "' +
# str(file_id) + "...")
# logger.warning(str({"log_event_type": "error", "title": "Unhandled error",
# "description": "Unhandled error while resolving file pointers to download file with ID: " + str(
# file_id) + "..."}))
# self.emit(QtCore.SIGNAL("showUnhandledException"), str(e)) # emit unhandled Exception
# logger.error(e)
i = 0
def upload_shard(self, shard, chapters, frame, file_name_ready_to_shard_upload):
self.uploadblocksize = 4096
def read_in_chunks(file_object, shard_size, rowposition, blocksize=self.uploadblocksize, chunks=-1, shard_index=None):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
i = 0
while chunks:
data = file_object.read(blocksize)
if not data:
break
yield data
i += 1
t1 = float(shard_size) / float((self.uploadblocksize))
if shard_size <= (self.uploadblocksize):
t1 = 1
percent_uploaded = int(round((100.0 * i) / t1))
logger.debug(i)
chunks -= 1
self.emit(SIGNAL("updateShardUploadProgress"), int(rowposition), percent_uploaded) # update progress bar in upload queue table
self.shard_upload_percent_list[shard_index] = percent_uploaded
self.emit(SIGNAL("refreshOverallProgress"), 0.1) # update overall progress bar
it = 0
contract_negotiation_tries = 0
while self.max_retries_negotiate_contract > contract_negotiation_tries:
contract_negotiation_tries += 1
# emit signal to add row to upload queue table
# self.emit(SIGNAL("addRowToUploadQueueTable"), "important", "information")
self.ui_single_file_upload.current_state.setText(
html_format_begin + "Adding shard " + str(
chapters) + " to file frame and getting contract..." + html_format_end)
# logger.warning('"log_event_type": "debug"')
logger.warning('"title": "Negotiating contract"')
logger.warning('"description": "Trying to negotiate storage \
contract for shard at inxed " + str(chapters) + "..."')
# logger.warning(str({"log_event_type": "debug", "title": "Negotiating contract",
# "description": "Trying to negotiate storage contract for shard at inxed " + str(chapters) + "..."}))
try:
frame_content = self.storj_engine.storj_client.frame_add_shard(shard, frame.id)
# Add items to shard queue table view
tablerowdata = {}
tablerowdata["farmer_address"] = frame_content["farmer"]["address"]
tablerowdata["farmer_port"] = frame_content["farmer"]["port"]
tablerowdata["hash"] = str(shard.hash)
tablerowdata["state"] = "Uploading..."
tablerowdata["token"] = frame_content["token"]
tablerowdata["shard_index"] = str(chapters)
# logger.warning('"log_event_type": "debug"')
logger.warning('"title": "Contract negotiated"')
logger.warning('"description": "Storage contract negotiated with: "' +
str(frame_content["farmer"]["address"]) + ":" +
str(frame_content["farmer"]["port"]))
# logger.warning(str({"log_event_type": "debug", "title": "Contract negotiated",
# "description": "Storage contract negotiated with: " + str(frame_content["farmer"]["address"] + ":" + str(frame_content["farmer"]["port"]))}))
self.emit(SIGNAL("addRowToUploadQueueTable"), tablerowdata) # add row to table
rowcount = self.ui_single_file_upload.shard_queue_table_widget.rowCount()
logger.debug(frame_content)
logger.debug(shard)
logger.debug(frame_content["farmer"]["address"])
farmerNodeID = frame_content["farmer"]["nodeID"]
url = "http://" + frame_content["farmer"]["address"] + ":" +\
str(frame_content["farmer"]["port"]) + "/shards/" +\
frame_content["hash"] + "?token=" +\
frame_content["token"]
logger.debug(url)
# files = {'file': open(file_path + '.part%s' % chapters)}
# headers = {'content-type: application/octet-stream', 'x-storj-node-id: ' + str(farmerNodeID)}
self.set_current_status("Uploading shard " + str(chapters + 1) + " to farmer...")
# begin recording exchange report
exchange_report = storj.model.ExchangeReport()
current_timestamp = int(time.time())
exchange_report.exchangeStart = str(current_timestamp)
exchange_report.farmerId = str(farmerNodeID)
exchange_report.dataHash = str(shard.hash)
shard_size = int(shard.size)
rowposition = rowcount
farmer_tries = 0
response = None
while self.max_retries_upload_to_same_farmer > farmer_tries:
farmer_tries += 1
try:
logger.warning(str({"log_event_type": "debug", "title": "Uploading shard",
"description": "Uploading shard at index " + str(shard.index) + " to " + str(
frame_content["farmer"]["address"] + ":" + str(frame_content["farmer"][
"port"]))}))
with open(self.parametrs.tmpPath + file_name_ready_to_shard_upload + '-' + str(chapters + 1),
'rb') as f:
response = requests.post(url, data=read_in_chunks(f, shard_size, rowposition, shard_index=chapters), timeout=1)
j = json.loads(str(response.content))
if (j["result"] == "The supplied token is not accepted"):
raise storj.exception.StorjFarmerError(
storj.exception.StorjFarmerError.SUPPLIED_TOKEN_NOT_ACCEPTED)
except storj.exception.StorjFarmerError as e:
# upload failed due to Farmer Failure
logger.error(e)
if str(e) == str(storj.exception.StorjFarmerError.SUPPLIED_TOKEN_NOT_ACCEPTED):
logger.error("The supplied token not accepted")
# print "Exception raised while trying to negitiate contract: " + str(e)
continue
except Exception as e:
self.emit(SIGNAL("updateUploadTaskState"), rowposition,
"First try failed. Retrying... (" + str(farmer_tries) + ")") # update shard upload state
# logger.warning('"log_event_type": "warning"')
logger.warning('"title": "Shard upload error"')
logger.warning('"description": "Error while uploading \
shard to: "' +
frame_content["farmer"]["address"] +
":" +
str(frame_content["farmer"]["port"]) +
" Retrying... (" + str(farmer_tries) +
")")
# logger.warning(str({"log_event_type": "warning", "title": "Shard upload error",
# "description": "Error while uploading shard to: " + str(
# frame_content["farmer"]["address"] + ":" + str(frame_content["farmer"][
# "port"])) + " Retrying... (" + str(farmer_tries) + ")"}))
logger.error(e)
continue
else:
self.emit(SIGNAL("incrementShardsProgressCounters")) # update already uploaded shards count
logger.warning(str({"log_event_type": "success", "title": "Uploading shard",
"description": "Shard uploaded successfully to " + str(
frame_content["farmer"]["address"] + ":" + str(frame_content["farmer"][
"port"]))}))
self.emit(SIGNAL("updateUploadTaskState"), rowposition,
"Uploaded!") # update shard upload state
logger.debug(str(self.all_shards_count) + "wszystkie" +
str(self.shards_already_uploaded) + "wyslane")
if int(self.all_shards_count) <= int(self.shards_already_uploaded + 1):
self.emit(SIGNAL("finishUpload")) # send signal to save to bucket after all files are uploaded
break
logger.debug(response.content)
j = json.loads(str(response.content))
if (j["result"] == "The supplied token is not accepted"):
raise storj.exception.StorjFarmerError(storj.exception.StorjFarmerError.SUPPLIED_TOKEN_NOT_ACCEPTED)
firstiteration = False
it += 1
except storj.exception.StorjBridgeApiError as e:
# upload failed due to Storj Bridge failure
logger.error("Exception raised while trying to negitiate \
contract: " + str(e))
# logger.warning('"log_event_type": "error"')
logger.warning('"title": "Bridge exception"')
logger.warning('"description": "Exception raised while trying \
to negotiate storage contract for shard at index\
"' + str(chapters))
# logger.warning(str({"log_event_type": "error", "title": "Bridge exception",
# "description": "Exception raised while trying to negitiate storage contract for shard at index " + str(
# chapters)}))
continue
except Exception as e:
# now send Exchange Report
# upload failed probably while sending data to farmer
logger.error("Error occured while trying to upload shard or\
negotiate contract. Retrying... " + str(e))
# logger.warning('"log_event_type": "error"')
logger.warning('"title": "Unhandled exception"')
logger.warning('"description": "Unhandled exception occured\
while trying to upload shard or negotiate \
contract for shard at index "' +
str(chapters) +
" . Retrying...")
# logger.warning(str({"log_event_type": "error", "title": "Unhandled exception",
# "description": "Unhandled exception occured while trying to upload shard or negotiate contract for shard at index " + str(chapters) + " . Retrying..."}))
current_timestamp = int(time.time())
exchange_report.exchangeEnd = str(current_timestamp)
exchange_report.exchangeResultCode = (exchange_report.FAILURE)
exchange_report.exchangeResultMessage = (exchange_report.STORJ_REPORT_UPLOAD_ERROR)
self.set_current_status("Sending Exchange Report for shard " + str(chapters + 1))
# self.storj_engine.storj_client.send_exchange_report(exchange_report) # send exchange report
continue
else:
# uploaded with success
current_timestamp = int(time.time())
# prepare second half of exchange heport
exchange_report.exchangeEnd = str(current_timestamp)
exchange_report.exchangeResultCode = (exchange_report.SUCCESS)
exchange_report.exchangeResultMessage = (exchange_report.STORJ_REPORT_SHARD_UPLOADED)
self.set_current_status("Sending Exchange Report for shard " + str(chapters + 1))
# logger.warning('"log_event_type": "debug"')
logger.debug('"title":"Shard added"')
logger.info('"description": "Shard "' + str(chapters + 1) +
" successfully added and exchange report sent.")
# logger.warning(str({"log_event_type": "debug", "title": "Shard added",
# "description": "Shard " + str(chapters + 1) + " successfully added and exchange report sent."}))
# self.storj_engine.storj_client.send_exchange_report(exchange_report) # send exchange report
break