def auth_valid(self, req): logger.info("%s authenticated %s", self.log_str(), str(self.user_id())) self.local_work_dir = None if not self.cfg().is_user_allowed(self.user_id()): raise RuntimeError("user not authorized") logger.debug("%s user %s allowed in role(s) %s", self.log_str(), str(self.user_id()), str(self.user_role())) # reattach older session if available old_sessions = self.remove_older_sessions() self.task = None self.detach = False for sess in old_sessions: if sess.task == None: continue self.task = sess.task self.detach = True self.local_work_dir = sess.local_work_dir self.reattach_older_session(sess) break if None == self.local_work_dir: self.local_work_dir = Utils.mkdtemp(prefix="sess_") logger.debug("%s created temporary folder %s", self.log_str(), str(self.local_work_dir)) else: logger.debug("%s reattached to old temporary folder %s", self.log_str(), str(self.local_work_dir)) req.set_secure_cookie(Session.cookie_name, self.sess_id, 1) req.redirect('/auth/storage?uid=' + self.user_id())
def run_verify(qlogger, msg_type, roles): qlogger.srvr_log(logging.INFO, "beginning run_verify") outdir = None cwd = os.getcwd() strio = StringIO.StringIO() try: root_path = os.path.dirname(cs_pkg_file) outdir = Utils.mkdtemp(prefix="verify_") if os.path.exists(root_path): root_path = os.path.split(root_path)[0] os.chdir(root_path) # otherwise we are running inside a packaged folder and resources are availale at cwd from circuitscape.verify import cs_verifyall testResult = cs_verifyall(out_path=outdir, ext_logger=qlogger, stream=strio) testsPassed = testResult.wasSuccessful() except Exception as e: qlogger.clnt_log("Unexpected error during verify.") qlogger.srvr_log(logging.WARNING, "Exception during verify: " + str(e)) testsPassed = False finally: os.chdir(cwd) Utils.rmdir(outdir) qlogger.clnt_log(strio.getvalue()) strio.close() qlogger.srvr_log(logging.INFO, "end run_verify") qlogger.send_result_msg(msg_type, {'complete': True, 'success': testsPassed})
def run_batch(qlogger, msg_type, roles, msg_data, work_dir, storage_creds, store_in_cloud, log_str): qlogger.srvr_log(logging.INFO, "beginning run_batch") cwd = os.getcwd() store = GoogleDriveStore(storage_creds, log_str) if (None != storage_creds) else None if store_in_cloud: batch_zip = msg_data batch_zip_name = os.path.splitext(store.to_file_name(batch_zip))[0] + '_output.zip' qlogger.clnt_log("Reading from cloud store: " + batch_zip) batch_folder = Utils.mkdtemp(prefix="batch_") batch_zip = store.copy_to_local(batch_zip, work_dir) Utils.uncompress_folder(batch_folder, batch_zip) os.remove(batch_zip) output_folder_zip = os.path.join(work_dir, batch_zip_name) else: batch_folder = msg_data qlogger.clnt_log("Running batch with all configurations (.ini files) under folder: " + batch_folder) output_root_folder = Utils.mkdtemp_if_exists(prefix="output", dir=batch_folder) if store_in_cloud: qlogger.clnt_log("Outputs would be uploaded to your cloud store as " + batch_zip_name) else: qlogger.clnt_log("Outputs would be stored under folder: " + output_root_folder) num_success = 0 num_failed = 0 # collect all configuration names config_files = [] for root, _dirs, files in os.walk(batch_folder): for file_name in files: if not file_name.endswith(".ini"): continue config_files.append(os.path.join(root, file_name)) num_configs = len(config_files) qlogger.clnt_log("Found " + str(num_configs) + " configuration files.") parallelize = True if (num_configs > CircuitscapeRunner.MAX_PARALLEL) else False batch_success = False if parallelize: pool = [] results = [] try: for config_file in config_files: root, file_name = os.path.split(config_file) qlogger.clnt_log("Loading configuration: " + config_file) cfg = CSConfig(config_file) os.chdir(root) qlogger.clnt_log("Verifying configuration...") if parallelize: # switch off parallization of each task if we are parallelizing batch cfg.parallelize = False (all_options_valid, message) = cfg.check() #qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid)) if all_options_valid: #qlogger.clnt_log("Verifying all paths are relative...") all_options_valid = cfg.are_all_paths_relative(); if not all_options_valid: message = "All file paths in configuration must be relative to location of configuration file." #qlogger.clnt_log("Verified configuration with result: " + str(all_options_valid)) if all_options_valid: qlogger.clnt_log("Verifying profile limits...") (all_options_valid, message) = CircuitscapeRunner.check_role_limits(roles, cfg, qlogger) if not all_options_valid: qlogger.send_error_msg(message) num_failed += 1 else: solver_failed = True cfgname = os.path.splitext(file_name)[0] output_folder = os.path.join(output_root_folder, cfgname) os.mkdir(output_folder) cfg.output_file = os.path.join(output_folder, os.path.basename(cfg.output_file)) outdir, _out_file = os.path.split(cfg.output_file) try: qlogger.clnt_log("Storing final configuration...") configFile = os.path.join(outdir, 'circuitscape.ini') cfg.write(configFile) if parallelize: if len(pool) >= CircuitscapeRunner.MAX_PARALLEL: p = pool.pop() p.join() result = results.pop() if result.value == 0: num_success += 1 else: num_failed += 1 result = Value('i', -1) p = Process(target=CircuitscapeRunner._run_compute, args=(configFile, qlogger, cfgname + ' => ', result)) pool.append(p) results.append(result) p.start() else: cs = Compute(configFile, qlogger) result, solver_failed = cs.compute() qlogger.clnt_log("Result: \n" + str(result)) except Exception as e: message = str(e) qlogger.send_error_msg(message) if not parallelize: if solver_failed: num_failed += 1 else: num_success += 1 if parallelize: qlogger.srvr_log(logging.DEBUG, "waiting for processes len=" + str(len(pool))) for idx in range(0, len(pool)): pool[idx].join() if results[idx].value == 0: num_success += 1 else: num_failed += 1 qlogger.srvr_log(logging.DEBUG, "Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed)) qlogger.clnt_log("Batch run done for " + str(len(config_files)) + " configuration files. Success: " + str(num_success) + ". Failures: " + str(num_failed)) if num_success > 0: if store_in_cloud: CircuitscapeRunner.upload_results(qlogger, output_root_folder, output_folder_zip, work_dir, msg_data, store, extract_folder_id=True) else: qlogger.clnt_log("Outputs under folder: " + output_root_folder) batch_success = True except Exception as e: qlogger.clnt_log("Unexpected error during batch run.") qlogger.srvr_log(logging.WARNING, "Exception during batch run: " + str(e)) finally: os.chdir(cwd) if store_in_cloud: try: Utils.rmdir(batch_folder) if os.path.exists(output_folder_zip): os.remove(output_folder_zip) except: qlogger.srvr_log(logging.ERROR, "Could not clear temporary files.") qlogger.srvr_log(logging.INFO, "end run_batch") qlogger.send_result_msg(msg_type, {'complete': True, 'success': batch_success})