def __process_url(self, url):
# Crawler config load
cfgCrawler = Config(
os.path.join(RAGPICKER_ROOT, 'config',
'crawler.conf')).get("clientConfig")
data = None
headers = {
'User-Agent':
cfgCrawler.get("browser_user_agent",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
'Accept-Language':
cfgCrawler.get("browser_accept_language", "en-US"),
}
# Save original socket
originalSocket = socket.socket
# Set TOR Socks proxy
commonutils.setTorProxy()
request = urllib2.Request(url, data, headers)
try:
url_dl = urllib2.urlopen(request, timeout=30).read()
except urllib2.HTTPError as e:
raise e
except urllib2.URLError as e:
raise e
except Exception, e:
raise IOError("Thread(" + self.processName +
") - %s - Error parsing %s" % (e, url))
def validate_authentication_header(request):
allowed_api = Config.get('API_AUTH_EXCEPTION').split(',')
logging.info(allowed_api)
logging.info(request.path)
if str(Config.get('CLIENT_AUTH_ENABLED')
) == 'True' and request.path not in allowed_api:
token = request.headers.get('google-id-token')
authorization = request.headers.get('Authorization')
client_id = Config.get('ALLOWED_CLIENT_ID')
logging.info("google_id_token => " + str(token))
logging.info("authorization => " + str(authorization))
logging.info("ALLOWED_CLIENT_ID => " + client_id)
if authorization is None and token is None:
raise ForbiddenException("Forbidden.")
if token:
logging.info("token: %s, client_id: %s" % (token, client_id))
validate_google_id_token(token, client_id)
elif authorization:
# authorization
validate_authentication_token(authorization)
else:
raise ForbiddenException("Forbidden.")
def validate(request):
if str(Config.get('CLIENT_AUTH_ENABLED')) == 'True':
token = request.headers.get('google_id_token')
client_id = Config.get('ALLOWED_CLIENT_ID')
if token is None:
if 'google_id_token' not in request.cookies:
raise ForbiddenException()
token = request.cookies.get("google_id_token")
try:
logging.info("token: %s, client_id: %s" % (token, client_id))
id_info = client.verify_id_token(token, client_id)
logging.info(id_info)
if Config.get('DOMAIN_CHECK') is True:
if id_info.get('hd') not in Config.get('ALLOWED_DOMAIN').split(
','):
raise ForbiddenException()
if id_info['iss'] not in [
'accounts.google.com', 'https://accounts.google.com'
]:
raise ForbiddenException()
except crypt.AppIdentityError:
# Invalid token
raise ForbiddenException()
def __process_url(self, url):
# Crawler config load
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
data = None
headers = {
'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
}
# Save original socket
originalSocket = socket.socket
# Set TOR Socks proxy
commonutils.setTorProxy()
request = urllib2.Request(url, data, headers)
try:
url_dl = urllib2.urlopen(request, timeout=30).read()
except urllib2.HTTPError as e:
raise e
except urllib2.URLError as e:
raise e
except Exception, e:
raise IOError("Thread(" + self.processName + ") - %s - Error parsing %s" % (e, url))
def get_stub(self, server):
"""
get stub of server
"""
server_cfg = Config().microserver.get(server)
if not server_cfg:
return None
self._ensure_server(server)
stub = getattr(str_to_class(server_cfg.get("pb2_grpc")),
server_cfg.get("stub"))
return stub(self.channels[server])
def getLine(line_id):
lines = Config.get("production-lines")
line = [line for line in lines if line["id"] == line_id]
if line:
return jsonify(line[0]), 200
else:
return "", 404
def health():
global system_shouldRun_False_Since
head = {}
process = psutil.Process(os.getpid())
memory_used = process.memory_info().rss
head["memory_used"] = memory_used
head["memory_used_human_readable"] = human_readable_bytes(memory_used)
healthy = False
if Config.get("system.shouldRun", False):
system_shouldRun_False_Since = None
head["message"] = "Running"
healthy = True
else:
if not system_shouldRun_False_Since:
system_shouldRun_False_Since = time.time()
# unhealthy if it takes more than 10m to shut down
if time.time() - system_shouldRun_False_Since > 600:
head["message"] = "Shutdown is taking more than ten minutes"
healthy = False
else:
head["message"] = "Shutting down"
healthy = True
return jsonify({"head": head, "data": healthy}), 200
def list_all_editions():
logging.info("Henter oversikt over alle utgaver")
try:
url = os.path.join(Config.get("nlb_api_url"), "editions")
params = {
'limit': '-1',
'start': '0',
'order': 'any',
'include-deleted': 'false',
'library': 'NLB',
'creative-work-metadata': 'none',
'editions-metadata': 'all',
}
headers = {
'Accept': "application/json",
'Content-Type': "application/json",
}
logging.info(url)
response = requests.request("GET",
url,
headers=headers,
params=params)
data = response.json()
editions = data["data"]
return editions
except Exception:
logging.exception(
"Klarte ikke returnere en liste over alle utgaver")
return []
def _process_url(self, url):
#Crawler config load
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
data = None
headers = {
'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
}
request = urllib2.Request(url, data, headers)
try:
url_dl = urllib2.urlopen(request, timeout=30).read()
except Exception, e:
raise IOError("Thread("+self.threadName+") - %s - Error parsing %s" % (e, url))
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(),
key=lambda PluginInfo: PluginInfo.name):
crawlerModul = pluginInfo.plugin_object
# Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file",
pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
try:
log.debug("Run Crawler: " + pluginInfo.name)
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
crawlerModul = pluginInfo.plugin_object
# Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file", pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
try:
log.debug("Run Crawler: " + pluginInfo.name)
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
def load(path):
singleton = FileLoader.getInstance()
paths = [f for f in glob.glob(path + "*.json", recursive=False)]
for path in paths:
with open(path) as file:
config = Config(json.load(file))
singleton.datasets[config.get('name')] = config
def _ensure_server(self, server):
"""
ensure connection of server is alive
"""
if server not in self.channels or not self.channels[server]:
server_cfg = Config().microserver.get(server)
channel = grpc.insecure_channel(server_cfg.get("server"))
self.channels[server] = channel
def process(dataRequest, dataResponse):
verb = dataRequest.getVerb()
config = dataRequest.getConfig()
context = dataRequest.getContext()
authentication = Config(config.get('authentication'))
authenticator = AuthenticationManager.get(
authentication.get('type'))(authentication)
authenticator.authenticate(context)
def upload(cls, content, filename):
gcs = storage.Client()
bucket = gcs.get_bucket(Config.get('DEFAULT_BUCKET'))
blob = bucket.blob(filename)
blob.upload_from_string(content, content_type="text/plain")
# The public URL can be used to directly access the uploaded file via HTTP.
return blob.public_url
def show( self ):
"""
Shows the dialog.
"""
# DB info
name, driver, version = core.database.Database.get_engine_info()
version = '.'.join( [str( x ) for x in version] )
data = "%i Magazine\n%i Vertragstypen\n%i Ausgaben\n%i Kunden\n%i Adressen\n%i Bankkonten\n%i Verträge" % ( core.database.Magazine.count(), core.database.Subscription.count(), core.database.Issue.count(), core.database.Customer.count(), core.database.Address.count(), core.database.Bankaccount.count(), core.database.Contract.count() )
self.builder.get_object( 'general_database_name_label' ).set_text( name )
self.builder.get_object( 'general_database_driver_label' ).set_text( driver )
self.builder.get_object( 'general_database_version_label' ).set_text( version )
self.builder.get_object( 'general_database_data_label' ).set_text( data )
self.builder.get_object( 'general_bic_open_entry' ).set_text( Config.get( "Autocompletion", "bic_file_de" ) )
self.builder.get_object( 'general_zipcode_open_entry' ).set_text( Config.get( "Autocompletion", "zipcode_file_de" ) )
self._magazinemanager.start_edit()
self.builder.get_object( "content" ).show_all()
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
#TOR Socks proxy
isTorEnabled = cfgCrawler.get("clientConfig").get("tor_enabled", False)
if isTorEnabled:
torProxyAdress = cfgCrawler.get("clientConfig").get("tor_proxyadress", "localhost")
torProxyPort = cfgCrawler.get("clientConfig").get("tor_proxyport", 9050)
# Route an HTTP request through the SOCKS proxy
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, torProxyAdress, torProxyPort)
socket.socket = socks.socksocket
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in crawlerPluginManager.getAllPlugins():
crawlerModul = pluginInfo.plugin_object
#Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file", pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
log.info("Run Crawler: " + pluginInfo.name)
try:
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
def debug_SmsCode(crawl):
if not crawl.debugMode:
return
userIn = input("输入短信验证码:")
import redis
import json
import random
from core.config import Config
root_path = get_root_path(__name__)
cfg = Config(root_path)
conn = redis.Redis(
host=cfg.get('REDIS_HOST'),
port=cfg.get('REDIS_PORT'),
db=cfg.get('REDIS_DB'))
#data = dict(smsvalue=userIn, token=self.token)
crawl.task["smsvalue"]=userIn
key="sy_request_{}".format(crawl.token)
value="动态密码_{}".format(userIn)
conn.lpush(key,value)
def edit_config(self, event):
row = self.list_config.GetSelectedRow()
if row is not wx.NOT_FOUND:
idx = int(self.list_config.GetTextValue(row, 0))
cfg = self.config[idx]
self.dict2field(cfg)
self.index = idx
self.show_pnl()
cfg = Config()
self.config = cfg.get('db')
self.dict2list()
def delete_config(self, event):
cfg = Config()
row = self.list_config.GetSelectedRow()
if row is not wx.NOT_FOUND:
idx = int(self.list_config.GetTextValue(row, 0))
conf = self.config
self.config = {'db': conf}
del self.config['db'][idx]
cfg.set(self.config)
cfg = Config()
self.config = cfg.get('db')
self.dict2list()
def __init__(self, parent, icon=None):
super(ConfigWindow, self).__init__(parent)
self.parent = parent
self.config = None
self.app = wx.GetApp()
self.index = -1
self.SetIcon(icon)
self.set_glyphs()
cfg = Config()
self.exit = True
if cfg.is_config('db'):
self.config = cfg.get('db')
self.dict2list()
def validate_google_id_token(token, client_id):
try:
id_info = client.verify_id_token(token, client_id)
logging.info(id_info)
logging.info("Domain Check")
logging.info(Config.get('DOMAIN_CHECK'))
if Config.get('DOMAIN_CHECK') == 'True':
if id_info.get('hd') not in Config.get('ALLOWED_DOMAIN').split(
', '):
raise ForbiddenException("Domain not allowed.")
if id_info['iss'] not in [
'accounts.google.com', 'https://accounts.google.com'
]:
raise ForbiddenException("Domain not allowed")
session['user_info'] = id_info
except crypt.AppIdentityError as e:
# Invalid token
raise ForbiddenException(e.message)
def getDirectories(structure):
if structure == "ranked":
return jsonify(Directory.dirs_ranked)
elif structure == "resolved":
dirs = {}
buffered_network_paths = Config.get("buffered_network_paths", {})
buffered_network_hosts = Config.get("buffered_network_hosts", {})
for dir in Directory.dirs_flat:
if isinstance(dir, str) and dir not in buffered_network_paths:
smb, file, unc = Filesystem.networkpath(Directory.dirs_flat[dir])
host = Filesystem.get_host_from_url(smb)
buffered_network_paths[dir] = smb
Config.set("buffered_network_paths." + dir, smb)
buffered_network_hosts[dir] = host
Config.set("buffered_network_hosts." + dir, host)
dirs[dir] = buffered_network_paths[dir]
return jsonify(dirs)
else:
return jsonify(Directory.dirs_flat)
def on_finished( self, rendering_results ):
template = self._template_env.get_template( "{}.template".format( "base" ) )
rendered_letters = [rendered_letter for rendered_letter in rendering_results if rendered_letter is not None]
if len( rendered_letters ) == 0:
raise ValueError( "No rendered letters" )
lco_template = Config.get("LetterRenderer", "lco_template")
if not lco_template:
lco_template = "a4paper"
rendered_document = template.render( {'lco_template': lco_template, 'prerendered_letters':rendered_letters} )
dirs = [paths.config('templates', 'latex'),
paths.config('images'),
paths.data('templates', 'latex'),
paths.data('images')]
textinputs = [dir for dir in dirs if os.path.exists(dir)]
pdflatex.compile_str(rendered_document, self._output_file, textinputs)
def printRagpickerInfos(print_options=False):
infolog = logging.getLogger("Info")
infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED))
infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED))
infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED))
infolog.info("")
pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"]
for place in pluginPlaces:
infolog.info(
color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA))
cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf'))
ragpickerPluginManager = PluginManager()
ragpickerPluginManager.setPluginPlaces([place])
ragpickerPluginManager.collectPlugins()
for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(),
key=lambda PluginInfo: PluginInfo.name):
options = cfg.get(pluginInfo.name)
if options.enabled:
infolog.info(
color("%s V%s %s", MAGENTA) %
(" |----[+] " + pluginInfo.name.ljust(25),
pluginInfo.version, pluginInfo.description))
if print_options:
for key, value in options.iteritems():
if key != "enabled":
infolog.info(
color(" |-- %s = %s",
MAGENTA) % (key, str(value)))
else:
infolog.info(
color("%s V%s %s", BLUE) %
(" |----[-] " + pluginInfo.name.ljust(25),
pluginInfo.version, pluginInfo.description))
infolog.info("")
infolog.info(
" %s %s" %
(color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE)))
infolog.info("")
checkVersion()
sys.stdout.flush()
def getLines(libraries):
assert isinstance(libraries, str), "libraries must be a string"
libraries = libraries.split(",")
libraries = set([library.lower() for library in libraries])
production_lines = []
for production_line in Config.get("production-lines"):
if "filters" not in production_line:
production_lines.append(production_line)
continue
filters = production_line["filters"]
production_line_libraries = set(
[library.lower() for library in filters.get("libraries", [])])
if "libraries" in filters and not libraries.intersection(
production_line_libraries):
continue
production_lines.append(production_line)
return jsonify(production_lines), 200
def run(self):
self.check_version()
cfg = Config()
if not cfg.is_config('db'):
msg = wx.MessageBox(
"As configurações não foram encontradas , será aberta uma janela para configurar.",
caption="Aviso",
style=wx.OK | wx.CENTRE | wx.ICON_ERROR,
parent=None,
)
if msg == wx.OK:
from views.config import ConfigWindow
cfg = ConfigWindow(None, icon=self.icon)
cfg.SetIcon(self.icon)
if cfg.ShowModal() == wx.OK:
self.run()
else:
cfg.config = cfg.get('db')
from views.main import MainView
window = MainView(self)
window.SetIcon(self.icon)
window.Show()
def printRagpickerInfos(print_options=False):
infolog = logging.getLogger("Info")
infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED))
infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED))
infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED))
infolog.info("")
pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"]
for place in pluginPlaces:
infolog.info(color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA))
cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf'))
ragpickerPluginManager = PluginManager()
ragpickerPluginManager.setPluginPlaces([place])
ragpickerPluginManager.collectPlugins()
for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
options = cfg.get(pluginInfo.name)
if options.enabled:
infolog.info(color("%s V%s %s", MAGENTA) % (" |----[+] " +
pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
if print_options:
for key, value in options.iteritems():
if key != "enabled":
infolog.info(color(" |-- %s = %s", MAGENTA) % (key, str(value)))
else:
infolog.info(color("%s V%s %s", BLUE) % (" |----[-] " +
pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
infolog.info("")
infolog.info(" %s %s" % (color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE)))
infolog.info("")
checkVersion()
sys.stdout.flush()
def mathML_transformation(self, mathml):
try:
url = Config.get("nlb_api_url") + "/stem/math"
payload = mathml
headers = {
'Accept': "application/json",
'Content-Type': 'text/html;charset=utf-8',
}
response = requests.request("POST",
url,
data=payload.encode('utf-8'),
headers=headers)
data = response.json()
html = data["generated"]["html"]
return html
except Exception:
self.report.warning(
"Error returning MathML transformation. Check STEM result")
# self.report.warning(traceback.format_exc(), preformatted=True)
element = etree.fromstring(mathml)
display_attrib = element.attrib["display"]
lang_attrib = element.attrib[
"{http://www.w3.org/XML/1998/namespace}lang"]
if display_attrib == "inline":
if lang_attrib == "nb" or lang_attrib == "nob" or lang_attrib == "nn":
return "<span>Matematisk formel</span>"
else:
return "<span>Mathematical formula</span>"
else:
if lang_attrib == "nb" or lang_attrib == "nob" or lang_attrib == "nn":
return "<p>Matematisk formel</p>"
else:
return "<p>Mathematical formula</p>"
def email(self, recipients, subject=None, should_email=True, should_message_slack=True, should_attach_log=True, should_escape_chars=True):
if not subject:
assert isinstance(self.title, str) or self.pipeline is not None, "either title or pipeline must be specified when subject is missing"
subject = self.title if self.title else self.pipeline.title
smtp = {
"host": Config.get("email.smtp.host", None),
"port": Config.get("email.smtp.port", None),
"user": Config.get("email.smtp.user", None),
"pass": Config.get("email.smtp.pass", None)
}
sender = Address(Config.get("email.sender.name", "undefined"), addr_spec=Config.get("email.sender.address", "*****@*****.**"))
# 0. Create attachment with complete log (including DEBUG statements)
if should_attach_log is True:
self.attachLog()
attachments = []
for m in self._messages["attachment"]:
smb, file, unc = Filesystem.networkpath(m["text"])
base_path = Filesystem.get_base_path(m["text"], self.pipeline.dir_base)
relpath = os.path.relpath(m["text"], base_path) if base_path else None
if m["text"].startswith(self.reportDir()):
relpath = os.path.relpath(m["text"], self.reportDir())
if not [a for a in attachments if a["unc"] == unc]:
attachments.append({
"title": "{}{}".format(relpath, ("/" if os.path.isdir(m["text"]) else "")),
"smb": smb,
"file": file,
"unc": unc,
"severity": m["severity"]
})
# Determine overall status
status = "INFO"
for message_type in self._messages:
for m in self._messages[message_type]:
if m["severity"] == "SUCCESS" and status in ["INFO"]:
status = "SUCCESS"
elif m["severity"] == "WARN" and status in ["INFO", "SUCCESS"]:
status = "WARN"
elif m["severity"] == "ERROR":
status = "ERROR"
try:
assert isinstance(smtp, dict), "smtp must be a dict"
assert isinstance(sender, Address), "sender must be a Address"
assert isinstance(recipients, str) or isinstance(recipients, list) or isinstance(recipients, tuple), "recipients must be a str, list or tuple"
assert isinstance(self.title, str) or self.pipeline and isinstance(self.pipeline.title, str), "title or pipeline.title must be a str"
if isinstance(recipients, str):
recipients = [recipients]
elif isinstance(recipients, tuple):
recipients = list(recipients)
if status == "ERROR":
for key in Config.get("administrators", default=[]):
if key not in recipients:
recipients.append(key)
# when testing, only allow e-mail addresses defined in the ALLOWED_EMAIL_ADDRESSES_IN_TEST env var
if Config.get("test"):
subject = "[test] " + subject
filtered_recipients = []
for recipient in recipients:
if recipient in Config.get("email.allowed_email_addresses_in_test"):
filtered_recipients.append(recipient)
recipients = filtered_recipients
# 1. join lines with severity SUCCESS/INFO/WARN/ERROR
markdown_text = []
for m in self._messages["message"]:
if should_escape_chars:
text = m['text'].replace("&", "&").replace("<", "<").replace(">", ">")
else:
text = m['text']
if m['preformatted'] is True:
markdown_text.append("<pre>{}</pre>".format(text))
elif m['severity'] != 'DEBUG':
markdown_text.append(text)
if attachments != [] or should_attach_log is True:
markdown_text.append("\n----\n")
markdown_text.append("\n# Lenker\n")
markdown_text.append("\n<ul style=\"list-style: none;\">")
# Pick icon and style for INFO-attachments
attachment_styles = {
"DEBUG": {
"icon": "🗎",
"style": ""
},
"INFO": {
"icon": "🛈",
"style": ""
},
"SUCCESS": {
"icon": "😄",
"style": "background-color: #bfffbf;"
},
"WARN": {
"icon": "😟",
"style": "background-color: #ffffbf;"
},
"ERROR": {
"icon": "ðŸ˜",
"style": "background-color: #ffbfbf;"
}
}
for attachment in attachments:
# UNC links seems to be preserved when viewed in Outlook.
# file: and smb: URIs are disallowed or removed.
# So these links will only work in Windows.
# If we need this to work cross-platform, we would have
# to map the network share paths to a web server so that
# the transfers go through http:. This could maybe be mapped
# using environment variables.
li = "<li>"
li += "<span style=\"vertical-align: middle; font-size: 200%;\">" + attachment_styles[attachment["severity"]]["icon"] + "</span> "
li += "<span style=\"vertical-align: middle; " + attachment_styles[attachment["severity"]]["style"] + "\">"
li += "<a href=\"file:///" + attachment["unc"] + "\">" + attachment["title"] + "</a> "
li += "<a href=\"" + attachment["smb"] + "\">" + self.img_string + "=\" alt=\"" + attachment["smb"] + "\"/>" + "</a> "
li += "</span>"
li += "</li>"
markdown_text.append(li)
markdown_text.append("</ul>\n")
label_string = ""
for label in self.pipeline.labels:
label_string += "[{}] ".format(label)
markdown_text.append("\n[{}] {} [{}] [status:{}]".format(self.pipeline.uid, label_string, self.pipeline.publication_format, status))
markdown_text = "\n".join(markdown_text)
# 2. parse string as Markdown and render as HTML
if should_escape_chars:
markdown_html = markdown.markdown(markdown_text, extensions=['markdown.extensions.fenced_code', 'markdown.extensions.codehilite'])
else:
markdown_html = markdown_text
markdown_html = '''<!DOCTYPE html>
<html>
<head>
<meta charset=\"utf-8\"/>
<title>''' + subject.replace("&", "&").replace("<", "<").replace(">", ">") + '''</title>
</head>
<body>
''' + markdown_html + '''
</body>
</html>
'''
if not should_email:
logging.info("[e-mail] Not sending email")
else:
# 3. build e-mail
msg = EmailMessage()
msg['Subject'] = re.sub(r"\s", " ", subject).strip()
msg['From'] = sender
msg['To'] = Report.emailStringsToAddresses(recipients)
msg.set_content(markdown_text)
msg.add_alternative(markdown_html, subtype="html")
logging.info("[e-mail] E-mail with subject '{}' will be sent to: {}".format(msg['Subject'], ", ".join(recipients)))
# 4. send e-mail
if smtp["host"] and smtp["port"]:
smtp_server = "{}:{}".format(smtp["host"], smtp["port"])
logging.info("[e-mail] SMTP server: {}".format(smtp_server))
with smtplib.SMTP(smtp_server) as s:
s.ehlo()
# s.starttls()
if smtp["user"] and smtp["pass"]:
s.login(smtp["user"], smtp["pass"])
else:
logging.debug("[e-mail] user/pass not configured")
logging.debug("[e-mail] sending…")
s.send_message(msg)
logging.debug("[e-mail] sending complete.")
else:
logging.warning("[e-mail] host/port not configured")
temp_md_obj = tempfile.NamedTemporaryFile(suffix=".md")
temp_html_obj = tempfile.NamedTemporaryFile(suffix=".html")
with open(temp_md_obj.name, "w") as f:
f.write(markdown_text)
logging.debug("[e-mail] markdown: {}".format(temp_md_obj.name))
with open(temp_html_obj.name, "w") as f:
f.write(markdown_html)
logging.debug("[e-mail] html: {}".format(temp_html_obj.name))
if should_attach_log is True:
path_mail = os.path.join(self.reportDir(), "email.html")
shutil.copy(temp_html_obj.name, path_mail)
self.mailpath = Filesystem.networkpath(path_mail)
else:
yesterday = datetime.now() - timedelta(1)
yesterday = str(yesterday.strftime("%Y-%m-%d"))
path_mail = os.path.join(self.pipeline.dir_reports, "logs", "dagsrapporter", yesterday, self.pipeline.uid + ".html")
shutil.copy(temp_html_obj.name, path_mail)
self.mailpath = Filesystem.networkpath(path_mail)
except AssertionError as e:
logging.error("[e-mail] " + str(e))
if not should_message_slack:
logging.warning("Not sending message to slack")
else:
# 5. send message to Slack
slack_attachments = []
for attachment in attachments:
color = None
if attachment["severity"] == "SUCCESS":
color = "good"
elif attachment["severity"] == "WARN":
color = "warning"
elif attachment["severity"] == "ERROR":
color = "danger"
slack_attachments.append({
"title_link": attachment["smb"],
"title": attachment["title"],
"fallback": attachment["title"],
"color": color
})
Slack.slack(text=subject, attachments=slack_attachments)
def emailPlainText(subject, message, recipients, should_email=True):
assert isinstance(subject, str), "subject must be a str, was: {}".format(type(subject))
assert isinstance(message, str), "message must be a str, was: {}".format(type(message))
if recipients is None:
logging.info("No recipients given, e-mail won't be sent: '" + subject + "'")
return
assert isinstance(recipients, str) or isinstance(recipients, list), (
"recipients must be a str or list, was: {}".format(type(recipients))
)
smtp = {
"host": Config.get("email.smtp.host", None),
"port": Config.get("email.smtp.port", None),
"user": Config.get("email.smtp.user", None),
"pass": Config.get("email.smtp.pass", None)
}
sender = Address(Config.get("email.sender.name", "undefined"), addr_spec=Config.get("email.sender.address", "*****@*****.**"))
if isinstance(recipients, str):
recipients = [recipients]
if not should_email:
logging.info("[e-mail] Not sending plain text email")
else:
if Config.get("test"):
subject = "[test] " + subject
filtered_recipients = []
for recipient in recipients:
if recipient in Config.get("email.allowed_email_addresses_in_test"):
filtered_recipients.append(recipient)
recipients = filtered_recipients
# 1. build e-mail
msg = EmailMessage()
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = Report.emailStringsToAddresses(recipients)
msg.set_content(message)
# 2. send e-mail
if not msg["To"]:
logging.warning("[e-mail] Email with subject \"{}\" has no recipients".format(subject))
else:
logging.info("[e-mail] Sending email with subject \"{}\" to: {}".format(subject, ", ".join(recipients)))
if isinstance(smtp["host"], str) and isinstance(smtp["port"], str):
with smtplib.SMTP(smtp["host"] + ":" + smtp["port"]) as s:
s.ehlo()
# s.starttls()
if smtp["user"] and smtp["pass"]:
s.login(smtp["user"], smtp["pass"])
else:
logging.debug("[e-mail] user/pass not configured")
logging.debug("[e-mail] sending…")
s.send_message(msg)
logging.debug("[e-mail] sending complete.")
else:
logging.warning("[e-mail] host/port not configured")
Slack.slack(text=subject, attachments=None)
try:
vxCage.exportVxCage(sha256, dumpDir)
except (Exception) as e:
log.error("Export-Error: %s" % e)
elif args['which'] == 'export':
log.info("Exporting {} {}".format(args['dirname'], args['sha256_file']))
dumpDir = os.path.normpath(args['dirname']) + os.sep
for sha256 in iterateSha256File(args['sha256_file'], args['json']):
try:
rapickerExport(sha256, dumpDir, database, vxCage)
except (Exception) as e:
log.error("Export-Error: %s" % e)
elif args['which'] == 'codeDBTag':
log.info("codeDBTag {} {}".format(args['tag'], args['sha256_file']))
config = cfgReporting.get("codeDB")
if not config.get("enabled"):
log.error("Sorry: CodeDB for Ragpicker is not enabled!")
sys.exit()
for sha256 in iterateSha256File(args['sha256_file'], args['json']):
try:
submitTag(sha256, args['tag'], config)
except (Exception) as e:
log.error("CodeDBTag-Error: %s" % e)
elif args['which'] == 'import':
log.info("Importing {}".format(args['dirname']))
impDir = os.path.normpath(args['dirname']) + os.sep
for file in os.listdir(impDir):
def update_list_of_books(format, identifiers, report=logging):
if format == "XHTML": # "XHTML" means e-book (we're reusing identifiers previously reserved for XHTML files)
catalog_changes_needed_filesize = 0
catalog_changes_needed_formatklar = 0
editions = Bibliofil.list_all_editions()
filesize_xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
filesize_xml += "<root>\n"
lines_formatklar = []
for book in editions:
identifier = book["identifier"]
library = book["library"]
if identifier in identifiers:
if library is None or library.upper() != "NLB":
logging.info(
"book_available: only NLB books should have distribution methods: {} / {}"
.format(identifier, library))
continue
epub_dir = os.path.join(Directory.dirs_flat["epub-ebook"],
book["identifier"])
has_epub = os.path.isdir(epub_dir)
if not has_epub:
continue
has_html = has_epub # generated based on the EPUB on the fly
# has_docx = has_epub # generated based on the EPUB on the fly
has_mobi = has_epub # generated based on the EPUB on the fly
size = 0
if has_epub:
for root, dirs, files in os.walk(epub_dir):
for file in files:
size += os.path.getsize(
os.path.join(root, file))
size_catalog = 0
if book["fileSize"] is not None:
size_catalog = int(book["fileSize"])
if size != size_catalog and size != 0:
logging.info(
f"Filesize for edition: {identifier} is not correct, will update. New filesize: {size} vs old {size_catalog}"
)
catalog_changes_needed_filesize += 1
filesize_xml += "<folder><name>{}</name><sizedata>{}</sizedata></folder>\n".format(
book["identifier"], size)
distribution_formats = Bibliofil.distribution_formats_epub(
has_epub, has_html, has_mobi, size)
distribution_formats_catalog = book["distribution"]
if distribution_formats != distribution_formats_catalog:
catalog_changes_needed_formatklar += 1
logging.info(
f"Distribution formats for edition {identifier} is not correct, will update. Distribution formats: {distribution_formats} vs old {distribution_formats_catalog}"
)
for distribution_format in distribution_formats:
lines_formatklar.append("{};{};{};{}".format(
identifier, distribution_format["name"],
distribution_format["format"],
distribution_format["method"]))
if catalog_changes_needed_filesize > 0:
logging.info(
f"{catalog_changes_needed_filesize} filesize catalog changes needed"
)
filesize_xml += "</root>\n"
logging.info(
"Sending filesize-e-mail to {} with content:".format(
Config.get("email.filesize.address")))
logging.info(filesize_xml)
Report.emailPlainText("filesize: ", filesize_xml,
Config.get("email.filesize.address"))
if catalog_changes_needed_formatklar > 0:
logging.info(
f"{catalog_changes_needed_formatklar} formatklar catalog changes needed"
)
logging.info("\n".join(lines_formatklar))
Report.emailPlainText("formatklar: ",
"\n".join(lines_formatklar),
Config.get("email.formatklar.address"))
if catalog_changes_needed_filesize == catalog_changes_needed_formatklar == 0:
logging.info(f"No catalog changes needed for format {format}")
else:
report.debug("unknown format: {}".format(format))
return
def book_available(format, identifier, report=logging, title=""):
library = None
# get the appropriate book identifier(s)
logging.info("{}/editions/{}?edition-metadata=all".format(
Config.get("nlb_api_url"), identifier))
response = requests.get("{}/editions/{}?edition-metadata=all".format(
Config.get("nlb_api_url"), identifier))
if response.status_code == 200:
data = response.json()['data']
library = data["library"]
if format == "XHTML": # "XHTML" means e-book (we're reusing identifiers previously reserved for XHTML files)
lines = []
epub_dir = os.path.join(Directory.dirs_flat["epub-ebook"],
identifier)
has_epub = os.path.isdir(epub_dir)
has_html = has_epub # generated based on the EPUB on the fly
# has_docx = has_epub # generated based on the EPUB on the fly
has_mobi = has_epub # generated based on the EPUB on the fly
size = 0
if has_epub:
for root, dirs, files in os.walk(epub_dir):
for file in files:
size += os.path.getsize(os.path.join(root, file))
filesize_xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
filesize_xml += "<root>\n"
filesize_xml += "<folder><name>{}</name><sizedata>{}</sizedata></folder>\n".format(
identifier, size)
filesize_xml += "</root>\n"
logging.info(
"Sending filesize-e-mail to {} with content:".format(
Config.get("email.filesize.address")))
logging.info(filesize_xml)
Report.emailPlainText("filesize: " + identifier, filesize_xml,
Config.get("email.filesize.address"))
distribution_formats = Bibliofil.distribution_formats_epub(
has_epub, has_html, has_mobi, size)
for distribution_format in distribution_formats:
lines.append("{};{};{};{}".format(
identifier, distribution_format["name"],
distribution_format["format"],
distribution_format["method"]))
if library is None or library.upper() != "NLB":
report.debug(
"book_available: only NLB books should have distribution methods: {} / {}"
.format(identifier, library))
lines = []
logging.info(
"Sending formatklar-e-mail to {} with content:".format(
Config.get("email.formatklar.address")))
logging.info("\n".join(lines))
Report.emailPlainText("formatklar: " + identifier,
"\n".join(lines),
Config.get("email.formatklar.address"))
if format == "DAISY 2.02":
lines = []
text = "Abklar;624595;08.12.2021;624595112021;November;"
#Abklar;nummer;dagensdato;boknummer-måned-år(mappenummer);Tittel(dc:title eller fra metadata for utgave)eks: Hjemmet, 44/2021
#string emailMessage = "Abklar;" + tidskrift + ";" + todaydatemail + ";" + finalDirectory + ";" + myTitle+";";
#Aviser
#string emailMessage = "Abklar;" + folderNameExp + ";" + todaydatemail + ";" + folderNameExp + Program.folderDatePerodika+ ";"+ ";";
dato = datetime.today().strftime('%d.%m.%Y')
mmyyyy = datetime.today().strftime('%m%Y')
identifier_short = identifier[:6]
text = f"Abklar;{identifier_short};{dato};{identifier};{title};"
logging.info("Sending abklar-e-mail to {} with content:".format(
Config.get("email.abklar.address")))
logging.info(text)
Report.emailPlainText("Abklar", text,
Config.get("email.abklar.address"))
else:
report.debug("book_available: unknown format: {}".format(format))
return
import logging
from core.config import Config
config = Config()
config.parse()
level = config.get('log.level')
logging.basicConfig(level=level)
logger = logging
os.makedirs(CACHEFOLDER)
PIDLOG = "%sautohome.pid" % CACHEFOLDER
Config.set("global", 'PID', PIDLOG)
init_log()
def main():
writePID(PIDLOG)
from cmd.weather import Weather
target = Weather('上海')
target.get()
if __name__ == "__main__":
init()
PIDLOG = Config.get('global', 'PID')
atexit.register(lambda: deletePID(PIDLOG))
params = sys.argv
if len(params) < 2:
Usage()
else:
if params[1] == "start":
if os.path.exists(PIDLOG):
logging.info("Program is running")
Usage()
else:
main()
elif params[1] == "stop":
stop(PIDLOG)
elif params[1] == "restart":
pass
def get_all_users(self, callback=None):
param = dict()
param['maxResults'] = 100
param['domain'] = Config.get('GOOGLE_DOMAIN')
param['orderBy'] = 'familyName'
self._get_all_users_inner(param, callback=callback)
def create_service(self):
self.service = discovery.build('admin', 'directory_v1', credentials=(
ServiceAccountCredentials.from_json_keyfile_name(Config.get('CREDENTIALS_FILE_PATH')) \
.create_scoped(Config.get('SCOPES')).create_delegated(Config.get('DELEGATED_USER'))
))
PIDLOG = "%sautohome.pid" % CACHEFOLDER
Config.set("global", "PID", PIDLOG)
init_log()
def main():
writePID(PIDLOG)
from cmd.weather import Weather
target = Weather("上海")
target.get()
if __name__ == "__main__":
init()
PIDLOG = Config.get("global", "PID")
atexit.register(lambda: deletePID(PIDLOG))
params = sys.argv
if len(params) < 2:
Usage()
else:
if params[1] == "start":
if os.path.exists(PIDLOG):
logging.info("Program is running")
Usage()
else:
main()
elif params[1] == "stop":
stop(PIDLOG)
elif params[1] == "restart":
pass