def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ Notes start time, then continually checks lock until give_up_after seconds have passed. When lock is free, starts the process, then unlocks the lock. Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ end = datetime.now() + timedelta(seconds=give_up_after) while (datetime.now() < end): if lock_is_free(db): try: #print("thread {0} is using the lock now".format(worker_hash)) worker_main(worker_hash, db) #print("thread {0} is done with the lock now".format(worker_hash)) except Exception as e: print(e) finally: unlock(db) return else: time.sleep(retry_interval) print("thread {0} has timed out")
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ total_time = 0 while total_time < give_up_after: if lock_is_free(db): try: worker_main(worker_hash, db) except Exception: delete_lock(db) break else: delete_lock(db) break else: time.sleep(retry_interval) total_time = total_time + retry_interval
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ try: db.insert_one({"_id": worker_hash, 'time': time.time()}) except Exception: db.update_one({"_id": worker_hash}, {'time': time.time()}) runtime = 0 while runtime < give_up_after: if lock_is_free(db, worker_hash): try: worker_main(worker_hash, db) db.delete_one({"_id": worker_hash}) break except Exception: pass runtime += retry_interval time.sleep(retry_interval)
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ total_wait_time = 0 while total_wait_time < give_up_after: if not (lock_is_free('worker', worker_hash, db) and lock('worker', worker_hash, db)): total_wait_time = total_wait_time + retry_interval sleep(retry_interval) continue try: worker_main(worker_hash, db) break except Exception as e: print(f'{worker_hash} encountered error: {e}') finally: unlock('worker', worker_hash, db) if total_wait_time >= give_up_after: print(f'{worker_hash} gave up after {total_wait_time}')
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ # Pass this worker over to set the lock needed to write to output.txt and # return the key/value needed to release the lock after the write to output.txt key_val_pair = lock_is_free(worker_hash, db) try: worker_main(worker_hash, db) #write the message to output.txt except Exception as e: print(str(e), "exception" ) #worker_main will crash sometimes, so this can log the crash #release the lock db.delete_one(key_val_pair) print("Release DB")
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ start_time = time.time() try: db.insert_one({"_id": 1}) except: while not lock_is_free(1): time.sleep(retry_interval) if time.time() - start_time >= give_up_after: return attempt_run_worker(worker_hash, give_up_after, db, retry_interval) return try: worker_main(worker_hash, db) db.delete_one({"_id": 1}) except: if not lock_is_free(1): db.delete_one({"_id": 1})
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ start_time = time.time() # active_count - currently running threads # By default, counts the main thread as well. count = active_count() - 1 # Register the worker in database if create_worker_entry(db, worker_hash, count, start_time): print(str(worker_hash) + " worker entered at " + str(start_time)) else: print("Error entering data for worker: " + str(worker_hash)) # Infite loop till worker time's out or fails/completes the task while True: current_time = time.time() # Check if thread's wait time has crossed alloted time limit if current_time - start_time > give_up_after: return "Time's up, I give up!" # Check if thread can can be serviced if lock_is_free(db, worker_hash, count): try: worker_main(worker_hash, db) if not status_complete(worker_hash, db): print("Error: Status to complete for worker: " + str(worker_hash)) except Exception as e: print(e) if not status_fail(worker_hash, db): print("Error: Status to fail for worker: " + str(worker_hash)) return "I failed!" return "I returned successfully!" # Sleep before trying again time.sleep(retry_interval)
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ if lock_is_free(): worker_main(worker_hash, db)
def attempt_run_worker(worker_hash, give_up_after, db, retry_interval): """ CHANGE MY IMPLEMENTATION, BUT NOT FUNCTION SIGNATURE Run the worker from worker.py by calling worker_main Args: worker_hash: a random string we will use as an id for the running worker give_up_after: if the worker has not run after this many seconds, give up db: an instance of MockDB retry_interval: continually poll the locking system after this many seconds until the lock is free, unless we have been trying for more than give_up_after seconds """ log.info("start") add_to_queue(db, worker_hash) log.info(f"added: {worker_hash} to queue") current_time = 0 while current_time < give_up_after: try: if lock_is_free(db, worker_hash): log.info(f"running job: {worker_hash}") worker_main(worker_hash, db) update_job(db, worker_hash, JobStatus.SUCCESS) write_line("output.txt", "") return except Exception as e: log.exception(f"Error occurred in worker: `{worker_hash}`.", e) update_job(db, worker_hash, JobStatus.FAILED, repr(e)) write_line("output.txt", "") return log.debug(f"{worker_hash}: retrying after {retry_interval} seconds") current_time += retry_interval time.sleep(retry_interval) log.info(f"Timeout reached for worker: {worker_hash}, giving up") update_job(db, worker_hash, JobStatus.FAILED, "Timeout reached") write_line("output.txt", "")