def fetch(): global stock config.logger.info("stock:fetch()") try: # fetch from server d = connection.getData(config.stock_url) config.logger.debug("stock: -> %s",d.status_code) j = d.json() if "success" in j and j["success"] and "result" in j and j["result"]: try: stock_semaphore.acquire(1) stock = j["result"] stock["retrieved"] = time.time() config.logger.debug("stock: -> retrieved") config.logger.debug("stock = %s"%stock) finally: if stock_semaphore.available() < 1: stock_semaphore.release(1) controller.reconnected() return True else: return False except Exception as e: import sys _, _, exc_tb = sys.exc_info() config.logger.error("stock: -> failure line %s: %s",exc_tb.tb_lineno,e) # import traceback # tr=traceback.format_exc() # aw = config.app_window # aw.qmc.adderror(str(tr)) controller.disconnect(remove_credentials = False, stop_queue = False) return False
def fetch(): global stock config.logger.info("stock:fetch()") try: stock_semaphore.acquire(1) # fetch from server d = connection.getData(config.stock_url) config.logger.debug("stock: -> %s",d.status_code) j = d.json() if "success" in j and j["success"] and "result" in j and j["result"]: stock = j["result"] stock["retrieved"] = time.time() config.logger.debug("stock: -> retrieved") config.logger.debug("stock = %s"%stock) controller.reconnected() return True else: return False except Exception as e: config.logger.error("stock: -> failure: %s",e) controller.disconnect(False) return False finally: if stock_semaphore.available() < 1: stock_semaphore.release(1)
def fetchServerUpdate(uuid, file=None): aw = config.app_window try: config.logger.debug( "sync:fetchServerUpdate() -> requesting update from server (file: %s)", file) last_modified = "" if file is not None: file_last_modified = util.getModificationDate(file) # if file modification data is newer than what we have in our sync cache (as the file was externally modified), # we update our sync cache # addSync(uuid,file_last_modified) # don't update the sync cache timestamp here as the changes might not have been submitted to the server yet else: file_last_modified = None if file_last_modified is not None: last_modified = "?modified_at=" + str( int(round(file_last_modified * 1000))) res = connection.getData(config.roast_url + "/" + uuid + last_modified) status = res.status_code if status == 204: # NO CONTENT: data on server is older then ours config.logger.debug( "sync:fetchServerUpdate() -> 204 data on server is older") #no newer data found on server, do nothing; controller.is_synced() might report an unsynced status # if file modification date is newer than what is known on the version from the server via the sync cache if file is not None and getSync(uuid) is None: config.logger.debug( "sync: -> file not in sync cache yet, we recuires to fetch the servers modification date and add the profile to the sync cache" ) # we recurse to get a 200 with the last_modification date from the server for this record to add it to the sync cache automatically fetchServerUpdate(uuid) pass elif status == 404: try: data = res.json() if "success" in data and not data["success"]: config.logger.debug( "sync:fetchServerUpdate() -> 404 roast record deleted on server" ) # data not found on server, remove UUID from sync cache delSync(uuid) # else there must be another cause of the 404 else: config.logger.debug( "sync:fetchServerUpdate() -> 404 server error") except: pass elif status == 200: # data on server is newer than ours => update with data from server config.logger.debug( "sync:fetchServerUpdate() -> 200 data on server is newer") data = res.json() if "result" in data: r = data["result"] config.logger.debug("sync: -> fetch: %s", r) if getSync(uuid) is None and "modified_at" in r: addSync(uuid, util.ISO86012epoch(r["modified_at"])) config.logger.debug( "sync: -> added profile automatically to sync cache") if file_last_modified is not None: config.logger.debug("sync: -> file last_modified date: %s", util.epoch2ISO8601(file_last_modified)) if "modified_at" in r and file_last_modified is not None and util.ISO86012epoch( r["modified_at"]) > file_last_modified: applyServerUpdates(r) else: config.logger.debug( "sync: -> data received from server was older!?") config.logger.debug( "sync: -> file last_modified epoch: %s", file_last_modified) config.logger.debug( "sync: -> server last_modified epoch: %s", util.ISO86012epoch(r["modified_at"])) config.logger.debug( "sync: -> server last_modified date: %s", r["modified_at"]) except requests.exceptions.ConnectionError as e: # more general: requests.exceptions.RequestException config.logger.debug( "sync: -> connection error in fetchServerUpdate(), disconnecting: %s", e) # we disconnect, but keep the queue running to let it automatically reconnect if possible controller.disconnect(remove_credentials=False, stop_queue=False) except Exception as e: import sys _, _, exc_tb = sys.exc_info() config.logger.error( "sync: Exception in fetchServerUpdate() in line %s: %s", exc_tb.tb_lineno, e) finally: aw.editgraphdialog = None # stop block opening the Roast Properties dialog while syncing from the server config.app_window.updatePlusStatusSignal.emit() # @UndefinedVariable
def run(self): global queue config.logger.debug("queue:run()") time.sleep(config.queue_start_delay) self.resume() # unpause self item = None while True: time.sleep(config.queue_task_delay) with self.state: if self.paused: self.state.wait() # block until notified config.logger.debug("queue: looking for next item to be fetched") if item is None: item = queue.get() time.sleep(config.queue_task_delay) config.logger.debug("queue: -> worker processing item: %s", item) iters = config.queue_retries + 1 keepTask = False while iters > 0: config.logger.debug("queue: -> remaining iterations: %s", iters) r = None try: r = connection.sendData(item["url"], item["data"], item["verb"]) r.raise_for_status() iters = 0 # successfully transmitted, we add/update the roasts UUID sync-cache if "roast_id" in item["data"] and "modified_at" in item[ "data"]: # we update the plus status icon if the given roast_id was not yet in the sync cache and thus new if sync.getSync(item["data"]["roast_id"]) is None: sync.addSync( item["data"]["roast_id"], util.ISO86012epoch( item["data"]["modified_at"])) config.app_window.updatePlusStatusSignal.emit( ) # @UndefinedVariable else: sync.addSync( item["data"]["roast_id"], util.ISO86012epoch( item["data"]["modified_at"])) except Exception as e: config.logger.debug("queue: -> task failed: %s", e) if r is not None: config.logger.debug("queue: -> status code %s", r.status_code) if r is not None and r.status_code == 401: # authentication failed controller.disconnect(remove_credentials=False) iters = 0 # we don't retry and keep the task item keepTask = True elif r is not None and r.status_code == 409: # conflict iters = 0 # we don't retry, but remove the task as it is faulty else: # 500 internal server error, 429 Client Error: Too Many Requests or others # something went wrong we don't mark this task as done and retry iters = iters - 1 time.sleep(config.queue_retry_delay) if not keepTask: # we call task_done to remove the item from the queue queue.task_done() item = None config.logger.debug("queue: -> task done") config.logger.debug("queue: end of run:while paused=%s", self.paused)
def run(self): global queue config.logger.debug("queue:run()") time.sleep(config.queue_start_delay) self.resume() # unpause self item = None while True: time.sleep(config.queue_task_delay) with self.state: if self.paused: self.state.wait() # block until notified config.logger.debug("queue: -> qsize: %s", queue.qsize()) config.logger.debug("queue: looking for next item to be fetched") try: if item is None: item = queue.get() time.sleep(config.queue_task_delay) config.logger.debug("queue: -> worker processing item: %s", item) iters = config.queue_retries + 1 while iters > 0: config.logger.debug("queue: -> remaining iterations: %s", iters) r = None try: # we upload only full roast records, or partial updates in case the are under sync (registered in the sync cache) if is_full_roast_record(item["data"]) or ( "roast_id" in item["data"] and sync.getSync(item["data"]["roast_id"])): controller.connect(clear_on_failure=False, interactive=False) r = connection.sendData(item["url"], item["data"], item["verb"]) r.raise_for_status() # successfully transmitted, we add/update the roasts UUID sync-cache iters = 0 self.addSyncItem(item) else: # partial sync updates for roasts not registered for syncing are ignored iters = 0 except ConnectionError as e: try: if controller.is_connected(): config.logger.debug( "queue: -> connection error, disconnecting: %s", e) # we disconnect, but keep the queue running to let it automatically reconnect if possible controller.disconnect(remove_credentials=False, stop_queue=False) except: pass # we don't change the iter, but retry to connect after a delay in the next iteration time.sleep(config.queue_retry_delay) except Exception as e: config.logger.debug("queue: -> task failed: %s", e) if r is not None: config.logger.debug("queue: -> status code %s", r.status_code) else: config.logger.debug("queue: -> no status code") if r is not None and r.status_code == 401: # authentication failed try: if controller.is_connected(): config.logger.debug( "queue: -> connection error, disconnecting: %s", e) # we disconnect, but keep the queue running to let it automatically reconnect if possible controller.disconnect( remove_credentials=False, stop_queue=False) except: pass # we don't change the iter, but retry to connect after a delay in the next iteration time.sleep(config.queue_retry_delay) elif r is not None and r.status_code == 409: # conflict iters = 0 # we don't retry, but remove the task as it is faulty else: # 500 internal server error, 429 Client Error: Too Many Requests, 404 Client Error: Not Found or others # something went wrong we don't mark this task as done and retry iters = iters - 1 time.sleep(config.queue_retry_delay) # we call task_done to remove the item from the queue queue.task_done() item = None config.logger.debug("queue: -> task done") config.logger.debug("queue: end of run:while paused=%s", self.paused) except Exception as e: pass