def view_json(self, FERListWidget, url, id, key): logger.info("[Field Extraction Rules]Viewing FER(s) as JSON") selecteditems = FERListWidget.selectedItems() if len(selecteditems) > 0: # make sure something was selected try: sumo = SumoLogic(id, key, endpoint=url, log_level=self.mainwindow.log_level) json_text = '' for selecteditem in selecteditems: for object in FERListWidget.currentcontent: if object['name'] == str(selecteditem.text()): item_id = object['id'] fer = sumo.get_fer(item_id) json_text = json_text + json.dumps( fer, indent=4, sort_keys=True) + '\n\n' self.json_window = ShowTextDialog('JSON', json_text, self.mainwindow.basedir) self.json_window.show() except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:\n\n' + str(e)) return else: self.mainwindow.errorbox('No FER selected.') return
def update_asset_category(category, count): log.debug(f"update_asset_category {category} {count}") session = db_obj.get_db_session() try: query_result = session.query(AssetCategory).filter( AssetCategory.category_name == category) if query_result.count() == 0: new_category = AssetCategory( category_name=category, asset_count=1, ) log.debug(f"adding asset category {category}") session.add(new_category) else: query_result.update({ AssetCategory.asset_count: query_result.first().asset_count + count, }) log.debug(f"updating asset category {category}") session.commit() except Exception as e: session.rollback log.exception(f"update_asset_category failed {e}") finally: if session is not None: session.close() log.debug(f"session closed")
def update_asset_info(asset_id, asset_info): log.debug(f"update_asset_info {asset_info}") session = db_obj.get_db_session() try: query_result = session.query(AssetInfo).filter( AssetInfo.asset_id == asset_id) if query_result.count() == 0: log.warn(f" asset not found {asset_info.asset_id}") else: old_category = query_result.first().asset_category query_result.update({ AssetInfo.asset_name: asset_info.asset_name, AssetInfo.asset_owner: asset_info.asset_owner, AssetInfo.asset_category: asset_info.asset_category, }) if old_category != asset_info.asset_category: AssetInfoDao.update_asset_category( asset_info.asset_category, 1) AssetInfoDao.update_asset_category(old_category, -1) session.commit() result = (session.query(AssetInfo).filter( AssetInfo.asset_id == asset_id).all()) except Exception as e: session.rollback log.exception(f"update_asset_info failed {e}") finally: if session is not None: session.close() log.debug(f"session closed") return result
def update_user_info(user_id, user_info): log.debug(f"update_user_info {user_info}") session = db_obj.get_db_session() try: query_result = session.query(UserInfo).filter(UserInfo.user_id == user_id) if query_result.count() == 0: log.warn(f" user not found {user_id}") else: query_result.update( { UserInfo.user_name: user_info.user_name, UserInfo.user_email: user_info.user_email, UserInfo.user_assets: user_info.user_assets, UserInfo.user_role: user_info.user_role, } ) session.commit() result = ( session.query(UserInfo).filter(UserInfo.user_id == user_id).all() ) except Exception as e: session.rollback log.exception(f"update_user_info failed {e}") finally: if session is not None: session.close() log.debug(f"session closed") return result
def run_ctk(func, cfg, org, space, appname, msg=None): """ This is a helper function to reduce code duplication when called by Chaos Toolkit. :param func: Fn[app] -> Optional[any]; A function which performs some actions using an app object. :param cfg: Dict[String, any]; Configuration information about the environment. :param org: String; Cloud Foundry organization containing the application. :param space: String; Cloud Foundry space containing the application. :param appname: String; Application in Cloud Foundry which is to be targeted. :param msg: Optional[String]; A message to display at the beginning of operations. :return: Dict[String, Any]; The serialized App object after all operations were performed. """ if cfg: Config().load_dict(cfg) if msg: logger.info(msg) try: app = App.discover(org, space, appname) if not app: raise FailedActivity("Error discovering app!") result = func(app) except SystemExit as err: logger.exception(err) raise FailedActivity(err) if result: raise FailedActivity( "Error performing operation! Function returned {}.".format(result)) logger.info("Done!") return app.serialize()
def run(program, base_query_file, query_files, temp_dir, threads, timeout=1000.0, answer_dir=None): q = multiprocessing.Queue() p = multiprocessing.Process(target=__run, args=( q, program, base_query_file, query_files, temp_dir, threads, answer_dir, )) p.start() p.join(timeout) p.kill() # kill lemondb with pkill subprocess.run(['pkill', '-f', 'lemondb']) if p.exitcode == 0: status, realtime, exception = q.get() if exception: logger.exception(exception) return status, realtime else: return "TLE", timeout
def predict_immunogenicity(self, pep, allele): custom_mask = allele_dict.get(allele, False) peptide = pep.upper() peplen = len(peptide) cterm = peplen - 1 score = 0 count = 0 if not custom_mask: mask_num = [0, 1, cterm] elif custom_mask: mask_str = custom_mask.split(",") mask_num = list(map(int, mask_str)) mask_num = [x - 1 for x in mask_num] if peplen > 9: pepweight = immunoweight[:5] + ((peplen - 9) * [0.30]) + immunoweight[5:] else: pepweight = immunoweight try: for pos in peptide: if pos not in list(immunoscale.keys()): logger.debug("{} {} {}".format(pos, pep, allele)) raise KeyError() elif count not in mask_num: score += pepweight[count] * immunoscale[pos] count += 1 else: count += 1 except Exception as ex: logger.exception(ex) return score
def copy_partition(self, PartitionListWidgetFrom, PartitionListWidgetTo, fromurl, fromid, fromkey, tourl, toid, tokey): logger.info("[Partitions]Copying Partition(s)") try: selecteditems = PartitionListWidgetFrom.selectedItems() if len(selecteditems) > 0: # make sure something was selected fromsumo = SumoLogic(fromid, fromkey, endpoint=fromurl, log_level=self.mainwindow.log_level) tosumo = SumoLogic(toid, tokey, endpoint=tourl, log_level=self.mainwindow.log_level) for selecteditem in selecteditems: for object in PartitionListWidgetFrom.currentcontent: if object['name'] == str(selecteditem.text()): item_id = object['id'] partitions_export = fromsumo.get_partition(item_id) status = tosumo.create_partition(partitions_export['name'], partitions_export['routingExpression'], analytics_tier=partitions_export['analyticsTier'], retention_period=partitions_export['retentionPeriod'], is_compliant=partitions_export['isCompliant']) self.update_partition_list(PartitionListWidgetTo, tourl, toid, tokey) return else: self.mainwindow.errorbox('You have not made any selections.') return except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:' + str(e)) self.update_partition_list(PartitionListWidgetTo, tourl, toid, tokey) return
def backup_partition(self, PartitionListWidget, url, id, key): logger.info("[Partitions]Backing Up Partition(s)") selecteditems = PartitionListWidget.selectedItems() if len(selecteditems) > 0: # make sure something was selected savepath = str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Backup Directory")) if os.access(savepath, os.W_OK): message = '' sumo = SumoLogic(id, key, endpoint=url, log_level=self.mainwindow.log_level) for selecteditem in selecteditems: for object in PartitionListWidget.currentcontent: if object['name'] == str(selecteditem.text()): item_id = object['id'] try: export = sumo.get_partition(item_id) savefilepath = pathlib.Path(savepath + r'/' + str(selecteditem.text()) + r'.partition.json') if savefilepath: with savefilepath.open(mode='w') as filepointer: json.dump(export, filepointer) message = message + str(selecteditem.text()) + r'.json' + '\n' except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:\n\n' + str(e)) return self.mainwindow.errorbox('Wrote files: \n\n' + message) else: self.mainwindow.errorbox("You don't have permissions to write to that directory") else: self.mainwindow.errorbox('No partition selected.') return
def copy_role(self, RoleListWidgetFrom, RoleListWidgetTo, UserListWidgetTo, fromurl, fromid, fromkey, tourl, toid, tokey): logger.info("[Users and Roles]Copying Role(s)") try: selecteditems = RoleListWidgetFrom.selectedItems() if len(selecteditems) > 0: # make sure something was selected fromsumo = SumoLogic(fromid, fromkey, endpoint=fromurl, log_level=self.mainwindow.log_level) tosumo = SumoLogic(toid, tokey, endpoint=tourl, log_level=self.mainwindow.log_level) for selecteditem in selecteditems: role_id = selecteditem.details['id'] role = fromsumo.get_role(role_id) status = tosumo.create_role(role) self.update_users_and_roles_lists(UserListWidgetTo, RoleListWidgetTo, tourl, toid, tokey) return else: self.mainwindow.errorbox('You have not made any selections.') return except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:' + str(e)) self.update_users_and_roles_lists(UserListWidgetTo, RoleListWidgetTo, tourl, toid, tokey) return
def view_role_json(self, RoleListWidget, url, id, key): logger.info("[Users and Roles]Viewing Roles(s) JSON") selecteditems = RoleListWidget.selectedItems() if len(selecteditems) > 0: # make sure something was selected try: sumo = SumoLogic(id, key, endpoint=url, log_level=self.mainwindow.log_level) json_text = '' for selecteditem in selecteditems: role_id = selecteditem.details['id'] role = sumo.get_role(role_id) json_text = json_text + json.dumps( role, indent=4, sort_keys=True) + '\n\n' self.json_window = ShowTextDialog('JSON', json_text, self.mainwindow.basedir) self.json_window.show() except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:\n\n' + str(e)) return else: self.mainwindow.errorbox('No role selected.') return
def ImportModel(self, request: ImportModelRequest, context: grpc.ServicerContext) -> Empty: try: self._manager.get_search(request.searchId).import_model(request.version, request.content) return Empty() except Exception as e: logger.exception(e) raise e
def AddLabeledExampleIds(self, request: AddLabeledExampleIdsRequest, context: grpc.ServicerContext) -> Empty: try: search = self._manager.get_search(request.searchId) examples = queue.Queue() exceptions = [] def get_examples(): try: for object_id in request.examples: example = search.retriever.get_object(object_id, [ATTR_DATA]) examples.put(LabeledExample(label=request.examples[object_id], content=example.content)) except Exception as e: exceptions.append(e) finally: examples.put(None) threading.Thread(target=get_examples, name='get-examples').start() search.add_labeled_examples(to_iter(examples)) if len(exceptions) > 0: raise exceptions[0] return Empty() except Exception as e: logger.exception(e) raise e
def GetSearches(self, request: SearchId, context: grpc.ServicerContext) -> Iterable[SearchInfo]: try: for search_id, metadata in self._manager.get_searches(): yield SearchInfo(searchId=search_id, metadata=metadata) except Exception as e: logger.exception(e) raise e
def StepInto(self): if self._InvocationStack.Count == 0: self._VMState |= VMState.HALT if self._VMState & VMState.HALT > 0 or self._VMState & VMState.FAULT > 0: logger.info("stopping because vm state is %s " % self._VMState) return op = None if self.CurrentContext.InstructionPointer >= len( self.CurrentContext.Script): op = RET else: op = self.CurrentContext.OpReader.ReadByte(do_ord=False) self.ops_processed += 1 try: if self._is_write_log: self.write_log("{} {}".format(self.ops_processed, ToName(op))) self.ExecuteOp(op, self.CurrentContext) except Exception as e: error_msg = "COULD NOT EXECUTE OP (%s): %s %s %s" % ( self.ops_processed, e, op, ToName(op)) self.write_log(error_msg) if self._exit_on_error: self._VMState |= VMState.FAULT else: logger.error(error_msg) logger.exception(e)
def load_objects(self, *args, **kwargs): qs = self.queryset(KlokanGoogleCSVLoader.collection, query={}) for doc in qs: try: wait = True asset_id = doc['id'] gdoc = self.collection.find_one({'id': asset_id}) if not gdoc: logger.debug('Searching {}'.format(asset_id)) links = self.query_duckgo('NSW+{}'.format(asset_id)) yield {'id': asset_id, 'google_links': links} else: wait = False logger.debug('Skip {}'.format(asset_id)) except Exception as e: logger.exception('Fail {} -> {}'.format(asset_id, str(e))) if wait: wait_sec = int(random.random() * 60) logger.debug('Waiting {} : {}s'.format(asset_id, wait_sec)) time.sleep(wait_sec)
async def socket_connect(self, stream, peer): addr, port = peer logger.info(f"Connection from {addr}:{port}") client = Client(addr, port, stream) self.clients.append(client) try: while True: try: msg = await stream.decode() except ModuleNotFoundError: logger.exception( "Client {client.name} sent bad message. Look at client logs." ) continue try: handler = dispatch_table[msg[Keys.TYPE]] except KeyError: logger.warning("Unknown message type.") continue await handler(self, client, msg) except ConnectionResetError: logger.info( f"Client {client.addr}:{client.port} forcefully disconnected.") except EOFError: logger.info( f"Client {client.addr}:{client.port} disconnected gracefully.") finally: self.disconnect_client(client)
def load_templates(raw): result = {} for k, v in raw.items(): try: if isinstance(v, dict): metadata = { k: v for k, v in v.items() if k not in ('choices', 'text') } if v.get('choices'): # Parse all choices for choice in v['choices']: result.setdefault(k, []).append( ResponseTemplate.from_metadata( k, choice, metadata)) else: # No choices, just an uppermost-level dict result[k] = ResponseTemplate.from_metadata(k, v, metadata) else: result[k] = ResponseTemplate.from_metadata(k, v) except Exception as e: log.warning(f"Error parsing the template of intent {k}: {v}") log.exception(e) # Sanity checks assert not any(k is None or v is None for k, v in result.items()), "There are None values" return result
def parentdircontentlist(self, ContentListWidget, url, id, key, radioselected, directorylabel): if ContentListWidget.updated: logger.info("Going Up One Content Folder") sumo = SumoLogic(id, key, endpoint=url) currentdir = ContentListWidget.currentdirlist[-1] if currentdir['id'] != 'TOP': parentdir = ContentListWidget.currentdirlist[-2] else: return try: if parentdir['id'] == 'TOP': ContentListWidget.currentdirlist = [] self.updatecontentlist(ContentListWidget, url, id, key, radioselected, directorylabel) return else: ContentListWidget.currentdirlist.pop() ContentListWidget.currentcontent = sumo.get_folder( parentdir['id']) self.updatecontentlist(ContentListWidget, url, id, key, radioselected, directorylabel) return except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:\n\n' + str(e)) return
def roll_call(form, field): # 'field' is ignored logger.debug("roll_call validation with data: {}".format(form.data)) data = {"create_hosts": False, "discard_hosts": False, "players": {}} for field in form: if type(field.data) is dict and "hosts" in field.data: data["players"][field.name] = field.data try: r = requests.post( "http://johann_conductor:5000/scores/{}/roll_call".format( form.name.data), json=data, ) if not r.ok: msg = "roll_call to Johann failed ({}):\n".format(r.reason) try: rj = r.json() msg += "\n".join(rj["messages"]) except Exception: pass finally: logger.warn(msg) raise ValidationError(msg) except Exception as e: msg = "Exception validating score '{}': {}".format( form.name.data, str(e)) logger.warn(msg) logger.exception(e) raise ValidationError(msg)
def add_user_info(user_info): log.debug(f"add_user_info {user_info}") session = db_obj.get_db_session() try: query_count = ( session.query(UserInfo) .filter(UserInfo.user_id == user_info.user_id) .count() ) if query_count > 0: log.warn(f" user already exists {user_info.user_id}") else: new_user = UserInfo( user_id=user_info.user_id, user_name=user_info.user_name, user_email=user_info.user_email, user_role=user_info.user_role, user_assets=user_info.user_assets, ) log.debug(f"add_user_info {new_user}") session.add(new_user) session.commit() result = new_user except Exception as e: session.rollback log.exception(f"add_user_info failed {e}") finally: if session is not None: session.close() log.debug(f"session closed") return result
def _insert_failure_categories(self): """Insert Exception categories into `failure` sourced from core.exceptions.__init__""" logger.info("Adding exceptions to database...") logger.info("Parsing exception to insert into database...") all_exceptions = db_exceptions engine = create_engine(self.db_url, echo=self.db_queries_log) Session = sessionmaker(bind=engine) session = Session() failure_types = [ { "failureid": err.code, "failuretype": err.err_type, "failuredescription": err.message } for err in all_exceptions ] try: f = [Failure(**failure) for failure in failure_types] session.add_all(f) session.commit() except SQLAlchemyError as e: session.rollback() os.remove(self.db_filename) raise SmartVADHIS2Exception(e) except Exception as e: session.rollback() os.remove(self.db_filename) logger.exception("Unknown exception occurred: %s", e) finally: session.close()
def working_volume(): volume_name = 'crazybox-' + str(uuid.uuid4()) logger.info("Creating new docker volume for working directory") try: try: client.volumes.create(name=volume_name) except APIError: logger.exception("Failed to create a docker volume") raise DockerError(str(e)) logger.info("New docker volume is created: %s", volume_name) yield volume_name finally: logger.info("Removing the docker volume: %s", volume_name) try: client.volumes.get(volume_name).remove(force=True) except NotFound: logger.warning("Failed to remove the docker volume, it doesn't exist") except APIError: logger.exception("Failed to remove the docker volume, try prune unused container first: ") ret = client.containers.prune() logger.info("SpaceReclaimed: %s, ContainersDeleted: %s", ret['SpaceReclaimed'], ret['ContainersDeleted']) ret = client.volumes.prune() logger.info("SpaceReclaimed: %s, VolumesDeleted: %s", ret['SpaceReclaimed'], ret['VolumesDeleted']) else: logger.info("Docker volume removed")
def get_key(key=None, filename='customer_key.json'): """Check API key if provided or read it from the file.""" if not key: logger.debug('No customer API key was provided') logger.debug('Reading the configuration file: %s', filename) json_config_file_name = path.join(path.dirname(__file__), 'data', filename) try: file = open(json_config_file_name, 'r') except FileNotFoundError as msg: print('Error: Cannot find `{}`.'.format(filename)) print('Please use `--key` optional argument.') logger.exception(msg) raise SystemExit(2) try: json_data = load(file) key = json_data['customer_key'] except KeyError as msg: print('Error: Cannot find data with in `%s`', filename) logger.exception(msg) raise SystemExit(1) finally: file.close() if not (isinstance(key, str) and len(key) == 36 and match('^[0-9A-Za-z-]*$', key)): print('Error: Key is invalid') raise SystemExit(1) return key
def command(self, command: str) -> None: """ parses and trigger the command action :param command: string :return: None """ try: if ' ' in command: self.active_hover = None x, y, face = self.add_rover(*self.validate_command(command)) self.active_hover = (x, y) return x, y, face x, y = self.active_hover for c in list(command): face = self.validate_command(c) x, y, face = self.move_hover(x, y, face) self.active_hover = None print( x, y, list(self.cardinals.keys())[list( self.cardinals.values()).index(face)]) except (ValueError, TypeError, KeyError) as e: self.active_hover = None logger.exception(e) except (OutOfBounds, IndexError) as e: self.active_hover = None logger.exception(e)
def get_or_create_facebook_user(page, facebook_id): """ Get or create the user. Create the user in the database if it does not exist yet. """ user = User.query.filter_by(facebook_id=facebook_id).first() if user is None: # Get user information profile = page.get_user_profile(facebook_id) # Create user user = User( first_name=profile.get("first_name", ""), last_name=profile.get("last_name", ""), gender=profile.get("gender", "NA"), locale=profile.get("locale", ""), timezone=profile.get("timezone", ""), facebook_id=profile.get("id", ""), ) # Save user in database try: DB.session.add(user) DB.session.commit() logger.info(f"User created: {user.id}") except SQLAlchemyError as error: logger.info(f"Failed to create user in the database") logger.exception(error) return user
def add_asset_info(asset_info): log.debug(f"add_asset_info {asset_info}") session = db_obj.get_db_session() try: count_result = (session.query(AssetInfo).filter( AssetInfo.asset_id == asset_info.asset_id).count()) if count_result > 0: log.warn(f" user already exists {asset_info.asset_id}") else: AssetInfoDao.update_asset_category(asset_info.asset_category, 1) new_asset = AssetInfo( asset_id=asset_info.asset_id, asset_name=asset_info.asset_name, asset_category=asset_info.asset_category, asset_owner=asset_info.asset_owner, ) log.debug(f"add_asset_info {new_asset}") session.add(new_asset) session.commit() result = new_asset except Exception as e: session.rollback log.exception(f"add_asset_info failed {e}") finally: if session is not None: session.close() log.debug(f"session closed") return result
def run(self): while True: task = self.invoke_queue.get() logger.info("SmartContractInvokeQueue Task: %s", str(task)) method_name, args = task logger.info("- method_name: %s, args: %s", method_name, task) logger.info("- queue size: %s", self.invoke_queue.qsize()) try: self.invoke_method(method_name, *args) except Exception as e: logger.exception(e) # Wait a few seconds logger.info("wait 60 seconds...") time.sleep(60) # Re-add the task to the queue logger.info("Re-adding task to queue") self.invoke_queue.put(task) finally: # Always mark task as done, because even on error it was done and re-added self.invoke_queue.task_done()
def copy_fers(self, FERListWidgetFrom, FERListWidgetTo, fromurl, fromid, fromkey, tourl, toid, tokey): logger.info("[Field Extraction Rules]Copying FER(s)") try: selecteditems = FERListWidgetFrom.selectedItems() if len(selecteditems) > 0: # make sure something was selected fromsumo = SumoLogic(fromid, fromkey, endpoint=fromurl, log_level=self.mainwindow.log_level) tosumo = SumoLogic(toid, tokey, endpoint=tourl, log_level=self.mainwindow.log_level) for selecteditem in selecteditems: for object in FERListWidgetFrom.currentcontent: if object['name'] == str(selecteditem.text()): item_id = object['id'] fer_export = fromsumo.get_fer(item_id) status = tosumo.create_fer( fer_export['name'], fer_export['scope'], fer_export['parseExpression']) self.update_FER_list(FERListWidgetTo, tourl, toid, tokey) return else: self.mainwindow.errorbox('You have not made any selections.') return except Exception as e: logger.exception(e) self.mainwindow.errorbox('Something went wrong:' + str(e)) self.update_FER_list(FERListWidgetTo, tourl, toid, tokey) return
def StepInto(self): if self._InvocationStack.Count == 0: logger.info("INVOCATION COUNT IS 0, HALT") self._VMState |= VMState.HALT if self._VMState & VMState.HALT > 0 or self._VMState & VMState.FAULT > 0: logger.info("stopping because vm state is %s " % self._VMState) return op = None if self.CurrentContext.InstructionPointer >= len( self.CurrentContext.Script): op = RET else: op = self.CurrentContext.OpReader.ReadByte(do_ord=False) # opname = ToName(op) # logger.info("____________________________________________________") # logger.info("[%s] %02x -> %s" % (self.ops_processed,int.from_bytes(op,byteorder='little'), opname)) # logger.info("-----------------------------------") self.ops_processed += 1 try: self.ExecuteOp(op, self.CurrentContext) except Exception as e: logger.error("COULD NOT EXECUTE OP: %s %s %s" % (e, op, ToName(op))) logger.exception(e)
def StepInto(self): if self._InvocationStack.Count == 0: self._VMState |= VMState.HALT if self._VMState & VMState.HALT > 0 or self._VMState & VMState.FAULT > 0: logger.info("stopping because vm state is %s " % self._VMState) return op = None if self.CurrentContext.InstructionPointer >= len(self.CurrentContext.Script): op = RET else: op = self.CurrentContext.OpReader.ReadByte(do_ord=False) self.ops_processed += 1 try: if self._is_write_log: self.write_log("{} {}".format(self.ops_processed, ToName(op))) self.ExecuteOp(op, self.CurrentContext) except Exception as e: error_msg = "COULD NOT EXECUTE OP (%s): %s %s %s" % (self.ops_processed, e, op, ToName(op)) self.write_log(error_msg) if self._exit_on_error: self._VMState |= VMState.FAULT else: logger.error(error_msg) logger.exception(e)
def wrapper(request, *args, **kwargs): try: res = func(request, *args, **kwargs) except Exception as e: logger.exception(e) request.setResponseCode(500) request.setHeader('Content-Type', 'application/json') return json.dumps({"error": str(e)}) return res