Пример #1
0
    def __process_url(self, url):
        # Crawler config load
        cfgCrawler = Config(
            os.path.join(RAGPICKER_ROOT, 'config',
                         'crawler.conf')).get("clientConfig")

        data = None
        headers = {
            'User-Agent':
            cfgCrawler.get("browser_user_agent",
                           "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
            'Accept-Language':
            cfgCrawler.get("browser_accept_language", "en-US"),
        }

        # Save original socket
        originalSocket = socket.socket
        # Set TOR Socks proxy
        commonutils.setTorProxy()

        request = urllib2.Request(url, data, headers)

        try:
            url_dl = urllib2.urlopen(request, timeout=30).read()
        except urllib2.HTTPError as e:
            raise e
        except urllib2.URLError as e:
            raise e
        except Exception, e:
            raise IOError("Thread(" + self.processName +
                          ") - %s - Error parsing %s" % (e, url))
Пример #2
0
def validate_authentication_header(request):

    allowed_api = Config.get('API_AUTH_EXCEPTION').split(',')
    logging.info(allowed_api)
    logging.info(request.path)

    if str(Config.get('CLIENT_AUTH_ENABLED')
           ) == 'True' and request.path not in allowed_api:

        token = request.headers.get('google-id-token')
        authorization = request.headers.get('Authorization')
        client_id = Config.get('ALLOWED_CLIENT_ID')

        logging.info("google_id_token => " + str(token))
        logging.info("authorization  => " + str(authorization))
        logging.info("ALLOWED_CLIENT_ID => " + client_id)

        if authorization is None and token is None:
            raise ForbiddenException("Forbidden.")

        if token:
            logging.info("token: %s, client_id: %s" % (token, client_id))
            validate_google_id_token(token, client_id)

        elif authorization:
            # authorization
            validate_authentication_token(authorization)
        else:
            raise ForbiddenException("Forbidden.")
Пример #3
0
def validate(request):

    if str(Config.get('CLIENT_AUTH_ENABLED')) == 'True':

        token = request.headers.get('google_id_token')
        client_id = Config.get('ALLOWED_CLIENT_ID')

        if token is None:
            if 'google_id_token' not in request.cookies:
                raise ForbiddenException()

            token = request.cookies.get("google_id_token")

        try:
            logging.info("token: %s, client_id: %s" % (token, client_id))
            id_info = client.verify_id_token(token, client_id)
            logging.info(id_info)

            if Config.get('DOMAIN_CHECK') is True:
                if id_info.get('hd') not in Config.get('ALLOWED_DOMAIN').split(
                        ','):
                    raise ForbiddenException()

            if id_info['iss'] not in [
                    'accounts.google.com', 'https://accounts.google.com'
            ]:
                raise ForbiddenException()

        except crypt.AppIdentityError:
            # Invalid token
            raise ForbiddenException()
Пример #4
0
 def __process_url(self, url):
     # Crawler config load
     cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
     
     data = None
     headers = {   
         'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
         'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
     }
     
     # Save original socket
     originalSocket = socket.socket
     # Set TOR Socks proxy
     commonutils.setTorProxy() 
            
     request = urllib2.Request(url, data, headers)
 
     try:
         url_dl = urllib2.urlopen(request, timeout=30).read()
     except urllib2.HTTPError as e:
         raise e
     except urllib2.URLError as e:    
         raise e
     except Exception, e:
         raise IOError("Thread(" + self.processName + ") - %s - Error parsing %s" % (e, url)) 
Пример #5
0
    def get_stub(self, server):
        """
        get stub of server
        """

        server_cfg = Config().microserver.get(server)
        if not server_cfg:
            return None

        self._ensure_server(server)

        stub = getattr(str_to_class(server_cfg.get("pb2_grpc")),
                       server_cfg.get("stub"))
        return stub(self.channels[server])
Пример #6
0
def getLine(line_id):
    lines = Config.get("production-lines")
    line = [line for line in lines if line["id"] == line_id]
    if line:
        return jsonify(line[0]), 200
    else:
        return "", 404
Пример #7
0
def health():
    global system_shouldRun_False_Since

    head = {}

    process = psutil.Process(os.getpid())
    memory_used = process.memory_info().rss
    head["memory_used"] = memory_used
    head["memory_used_human_readable"] = human_readable_bytes(memory_used)

    healthy = False
    if Config.get("system.shouldRun", False):
        system_shouldRun_False_Since = None
        head["message"] = "Running"
        healthy = True

    else:
        if not system_shouldRun_False_Since:
            system_shouldRun_False_Since = time.time()

        # unhealthy if it takes more than 10m to shut down
        if time.time() - system_shouldRun_False_Since > 600:
            head["message"] = "Shutdown is taking more than ten minutes"
            healthy = False
        else:
            head["message"] = "Shutting down"
            healthy = True

    return jsonify({"head": head, "data": healthy}), 200
Пример #8
0
 def list_all_editions():
     logging.info("Henter oversikt over alle utgaver")
     try:
         url = os.path.join(Config.get("nlb_api_url"), "editions")
         params = {
             'limit': '-1',
             'start': '0',
             'order': 'any',
             'include-deleted': 'false',
             'library': 'NLB',
             'creative-work-metadata': 'none',
             'editions-metadata': 'all',
         }
         headers = {
             'Accept': "application/json",
             'Content-Type': "application/json",
         }
         logging.info(url)
         response = requests.request("GET",
                                     url,
                                     headers=headers,
                                     params=params)
         data = response.json()
         editions = data["data"]
         return editions
     except Exception:
         logging.exception(
             "Klarte ikke returnere en liste over alle utgaver")
         return []
Пример #9
0
 def _process_url(self, url):
     #Crawler config load
     cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
     
     data = None
     headers = {   
         'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
         'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
     }
     
     request = urllib2.Request(url, data, headers)
 
     try:
         url_dl = urllib2.urlopen(request, timeout=30).read()
     except Exception, e:
         raise IOError("Thread("+self.threadName+") - %s - Error parsing %s" % (e, url)) 
Пример #10
0
def runCrawler():
    mapURL = {}
    cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))

    # Build the PluginManager
    crawlerPluginManager = PluginManager()
    crawlerPluginManager.setPluginPlaces(["crawler"])
    crawlerPluginManager.collectPlugins()

    # Trigger run from the "Crawler" plugins
    for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(),
                             key=lambda PluginInfo: PluginInfo.name):
        crawlerModul = pluginInfo.plugin_object

        # Config for crawler module
        try:
            options = cfgCrawler.get(pluginInfo.name)
            crawlerModul.set_options(options)
        except Exception:
            log.error("Crawler module %s not found in configuration file",
                      pluginInfo.name)

        # If the crawler module is disabled in the config, skip it.
        if not options.enabled:
            continue

        try:
            log.debug("Run Crawler: " + pluginInfo.name)
            returnMap = crawlerModul.run()
            mapURL.update(returnMap)
        except Exception as e:
            log.error('Error (%s) in %s', e, pluginInfo.name)

    return mapURL
Пример #11
0
def runCrawler():
	mapURL = {}
	cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))	
	
	# Build the PluginManager
	crawlerPluginManager = PluginManager()
	crawlerPluginManager.setPluginPlaces(["crawler"])
	crawlerPluginManager.collectPlugins()
	
	# Trigger run from the "Crawler" plugins
	for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
		crawlerModul = pluginInfo.plugin_object
		
		# Config for crawler module
		try:
			options = cfgCrawler.get(pluginInfo.name)
			crawlerModul.set_options(options)
		except Exception:
			log.error("Crawler module %s not found in configuration file", pluginInfo.name)  
			
		# If the crawler module is disabled in the config, skip it.
		if not options.enabled:
			continue
		
		try:
			log.debug("Run Crawler: " + pluginInfo.name)
			returnMap = crawlerModul.run()
			mapURL.update(returnMap)
		except Exception as e:
			log.error('Error (%s) in %s', e, pluginInfo.name)
	
	return mapURL
Пример #12
0
    def load(path):
        singleton = FileLoader.getInstance()
        paths = [f for f in glob.glob(path + "*.json", recursive=False)]

        for path in paths:
            with open(path) as file:
                config = Config(json.load(file))
                singleton.datasets[config.get('name')] = config
Пример #13
0
 def _ensure_server(self, server):
     """
     ensure connection of server is alive
     """
     if server not in self.channels or not self.channels[server]:
         server_cfg = Config().microserver.get(server)
         channel = grpc.insecure_channel(server_cfg.get("server"))
         self.channels[server] = channel
Пример #14
0
    def process(dataRequest, dataResponse):
        verb = dataRequest.getVerb()
        config = dataRequest.getConfig()
        context = dataRequest.getContext()

        authentication = Config(config.get('authentication'))
        authenticator = AuthenticationManager.get(
            authentication.get('type'))(authentication)
        authenticator.authenticate(context)
Пример #15
0
    def upload(cls, content, filename):
        gcs = storage.Client()
        bucket = gcs.get_bucket(Config.get('DEFAULT_BUCKET'))
        blob = bucket.blob(filename)

        blob.upload_from_string(content, content_type="text/plain")

        # The public URL can be used to directly access the uploaded file via HTTP.
        return blob.public_url
Пример #16
0
    def show( self ):
        """
        Shows the dialog.
        """
        # DB info
        name, driver, version = core.database.Database.get_engine_info()
        version = '.'.join( [str( x ) for x in version] )
        data = "%i Magazine\n%i Vertragstypen\n%i Ausgaben\n%i Kunden\n%i Adressen\n%i Bankkonten\n%i Verträge" % ( core.database.Magazine.count(), core.database.Subscription.count(), core.database.Issue.count(), core.database.Customer.count(), core.database.Address.count(), core.database.Bankaccount.count(), core.database.Contract.count() )
        self.builder.get_object( 'general_database_name_label' ).set_text( name )
        self.builder.get_object( 'general_database_driver_label' ).set_text( driver )
        self.builder.get_object( 'general_database_version_label' ).set_text( version )
        self.builder.get_object( 'general_database_data_label' ).set_text( data )

        self.builder.get_object( 'general_bic_open_entry' ).set_text( Config.get( "Autocompletion", "bic_file_de" ) )
        self.builder.get_object( 'general_zipcode_open_entry' ).set_text( Config.get( "Autocompletion", "zipcode_file_de" ) )

        self._magazinemanager.start_edit()

        self.builder.get_object( "content" ).show_all()
Пример #17
0
def runCrawler():
	mapURL = {}
	cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
	
	#TOR Socks proxy
	isTorEnabled = cfgCrawler.get("clientConfig").get("tor_enabled", False)
	
	if isTorEnabled:
		torProxyAdress = cfgCrawler.get("clientConfig").get("tor_proxyadress", "localhost")
		torProxyPort = cfgCrawler.get("clientConfig").get("tor_proxyport", 9050)
		# Route an HTTP request through the SOCKS proxy 
		socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, torProxyAdress, torProxyPort)
		socket.socket = socks.socksocket			
	
	# Build the PluginManager
	crawlerPluginManager = PluginManager()
	crawlerPluginManager.setPluginPlaces(["crawler"])
	crawlerPluginManager.collectPlugins()
	
	# Trigger run from the "Crawler" plugins
	for pluginInfo in crawlerPluginManager.getAllPlugins():
		crawlerModul = pluginInfo.plugin_object
		
		#Config for crawler module
		try:
			options = cfgCrawler.get(pluginInfo.name)
			crawlerModul.set_options(options)
		except Exception:
			log.error("Crawler module %s not found in configuration file", pluginInfo.name)  
			
		# If the crawler module is disabled in the config, skip it.
		if not options.enabled:
			continue
		
		log.info("Run Crawler: " + pluginInfo.name)
		
		try:
			returnMap = crawlerModul.run()
			mapURL.update(returnMap)
		except Exception as e:
			log.error('Error (%s) in %s', e, pluginInfo.name)
			
	return mapURL
Пример #18
0
def debug_SmsCode(crawl):
    if not crawl.debugMode:
        return
    userIn = input("输入短信验证码:")
    import redis
    import json
    import random
    from core.config import Config
    root_path = get_root_path(__name__)
    cfg = Config(root_path)
    conn = redis.Redis(
        host=cfg.get('REDIS_HOST'),
        port=cfg.get('REDIS_PORT'),
        db=cfg.get('REDIS_DB'))
    #data = dict(smsvalue=userIn, token=self.token)
    crawl.task["smsvalue"]=userIn
    key="sy_request_{}".format(crawl.token)
    value="动态密码_{}".format(userIn)
    conn.lpush(key,value)
Пример #19
0
 def edit_config(self, event):
     row = self.list_config.GetSelectedRow()
     if row is not wx.NOT_FOUND:
         idx = int(self.list_config.GetTextValue(row, 0))
         cfg = self.config[idx]
         self.dict2field(cfg)
         self.index = idx
         self.show_pnl()
     cfg = Config()
     self.config = cfg.get('db')
     self.dict2list()
Пример #20
0
 def delete_config(self, event):
     cfg = Config()
     row = self.list_config.GetSelectedRow()
     if row is not wx.NOT_FOUND:
         idx = int(self.list_config.GetTextValue(row, 0))
         conf = self.config
         self.config = {'db': conf}
         del self.config['db'][idx]
         cfg.set(self.config)
     cfg = Config()
     self.config = cfg.get('db')
     self.dict2list()
Пример #21
0
 def __init__(self, parent, icon=None):
     super(ConfigWindow, self).__init__(parent)
     self.parent = parent
     self.config = None
     self.app = wx.GetApp()
     self.index = -1
     self.SetIcon(icon)
     self.set_glyphs()
     cfg = Config()
     self.exit = True
     if cfg.is_config('db'):
         self.config = cfg.get('db')
         self.dict2list()
Пример #22
0
def validate_google_id_token(token, client_id):
    try:

        id_info = client.verify_id_token(token, client_id)

        logging.info(id_info)
        logging.info("Domain Check")
        logging.info(Config.get('DOMAIN_CHECK'))

        if Config.get('DOMAIN_CHECK') == 'True':
            if id_info.get('hd') not in Config.get('ALLOWED_DOMAIN').split(
                    ', '):
                raise ForbiddenException("Domain not allowed.")

        if id_info['iss'] not in [
                'accounts.google.com', 'https://accounts.google.com'
        ]:
            raise ForbiddenException("Domain not allowed")

        session['user_info'] = id_info

    except crypt.AppIdentityError as e:
        # Invalid token
        raise ForbiddenException(e.message)
Пример #23
0
def getDirectories(structure):
    if structure == "ranked":
        return jsonify(Directory.dirs_ranked)

    elif structure == "resolved":
        dirs = {}

        buffered_network_paths = Config.get("buffered_network_paths", {})
        buffered_network_hosts = Config.get("buffered_network_hosts", {})

        for dir in Directory.dirs_flat:
            if isinstance(dir, str) and dir not in buffered_network_paths:
                smb, file, unc = Filesystem.networkpath(Directory.dirs_flat[dir])
                host = Filesystem.get_host_from_url(smb)
                buffered_network_paths[dir] = smb
                Config.set("buffered_network_paths." + dir, smb)
                buffered_network_hosts[dir] = host
                Config.set("buffered_network_hosts." + dir, host)
            dirs[dir] = buffered_network_paths[dir]

        return jsonify(dirs)

    else:
        return jsonify(Directory.dirs_flat)
Пример #24
0
    def on_finished( self, rendering_results ):
        template = self._template_env.get_template( "{}.template".format( "base" ) )
        rendered_letters = [rendered_letter for rendered_letter in rendering_results if rendered_letter is not None]
        if len( rendered_letters ) == 0:
            raise ValueError( "No rendered letters" )
        lco_template = Config.get("LetterRenderer", "lco_template")
        if not lco_template:
            lco_template = "a4paper"
        rendered_document = template.render( {'lco_template': lco_template, 'prerendered_letters':rendered_letters} )

        dirs = [paths.config('templates', 'latex'),
                paths.config('images'),
                paths.data('templates', 'latex'),
                paths.data('images')]
        textinputs = [dir for dir in dirs if os.path.exists(dir)]
        pdflatex.compile_str(rendered_document, self._output_file, textinputs)
Пример #25
0
def printRagpickerInfos(print_options=False):
    infolog = logging.getLogger("Info")
    infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED))
    infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED))
    infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED))
    infolog.info("")

    pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"]

    for place in pluginPlaces:
        infolog.info(
            color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA))
        cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf'))
        ragpickerPluginManager = PluginManager()
        ragpickerPluginManager.setPluginPlaces([place])
        ragpickerPluginManager.collectPlugins()

        for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(),
                                 key=lambda PluginInfo: PluginInfo.name):
            options = cfg.get(pluginInfo.name)

            if options.enabled:
                infolog.info(
                    color("%s V%s   %s", MAGENTA) %
                    ("              |----[+] " + pluginInfo.name.ljust(25),
                     pluginInfo.version, pluginInfo.description))
                if print_options:
                    for key, value in options.iteritems():
                        if key != "enabled":
                            infolog.info(
                                color("                    |-- %s = %s",
                                      MAGENTA) % (key, str(value)))
            else:
                infolog.info(
                    color("%s V%s   %s", BLUE) %
                    ("              |----[-] " + pluginInfo.name.ljust(25),
                     pluginInfo.version, pluginInfo.description))

    infolog.info("")
    infolog.info(
        "            %s %s" %
        (color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE)))
    infolog.info("")

    checkVersion()

    sys.stdout.flush()
Пример #26
0
def getLines(libraries):
    assert isinstance(libraries, str), "libraries must be a string"
    libraries = libraries.split(",")
    libraries = set([library.lower() for library in libraries])

    production_lines = []
    for production_line in Config.get("production-lines"):
        if "filters" not in production_line:
            production_lines.append(production_line)
            continue

        filters = production_line["filters"]
        production_line_libraries = set(
            [library.lower() for library in filters.get("libraries", [])])
        if "libraries" in filters and not libraries.intersection(
                production_line_libraries):
            continue

        production_lines.append(production_line)

    return jsonify(production_lines), 200
Пример #27
0
 def run(self):
     self.check_version()
     cfg = Config()
     if not cfg.is_config('db'):
         msg = wx.MessageBox(
             "As configurações não foram encontradas , será aberta uma janela para configurar.",
             caption="Aviso",
             style=wx.OK | wx.CENTRE | wx.ICON_ERROR,
             parent=None,
         )
         if msg == wx.OK:
             from views.config import ConfigWindow
             cfg = ConfigWindow(None, icon=self.icon)
             cfg.SetIcon(self.icon)
             if cfg.ShowModal() == wx.OK:
                 self.run()
     else:
         cfg.config = cfg.get('db')
         from views.main import MainView
         window = MainView(self)
         window.SetIcon(self.icon)
         window.Show()
Пример #28
0
def printRagpickerInfos(print_options=False):
	infolog = logging.getLogger("Info")
	infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED))
	infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED))
	infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED))
	infolog.info("")
	
	pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"]
	
	for place in pluginPlaces:
		infolog.info(color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA))
		cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf'))
		ragpickerPluginManager = PluginManager()
		ragpickerPluginManager.setPluginPlaces([place])
		ragpickerPluginManager.collectPlugins()
		
		for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
			options = cfg.get(pluginInfo.name) 
			
			if options.enabled:
				infolog.info(color("%s V%s   %s", MAGENTA) % ("              |----[+] " + 
						pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
				if print_options:
					for key, value in options.iteritems():
						if key != "enabled":
							infolog.info(color("                    |-- %s = %s", MAGENTA) % (key, str(value)))
			else:
				infolog.info(color("%s V%s   %s", BLUE) % ("              |----[-] " + 
						pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
	
	infolog.info("")
	infolog.info("            %s %s" % (color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE)))
	infolog.info("")
	
	checkVersion()
				
	sys.stdout.flush()
Пример #29
0
    def mathML_transformation(self, mathml):
        try:
            url = Config.get("nlb_api_url") + "/stem/math"
            payload = mathml
            headers = {
                'Accept': "application/json",
                'Content-Type': 'text/html;charset=utf-8',
            }

            response = requests.request("POST",
                                        url,
                                        data=payload.encode('utf-8'),
                                        headers=headers)

            data = response.json()

            html = data["generated"]["html"]
            return html
        except Exception:
            self.report.warning(
                "Error returning MathML transformation. Check STEM result")
            #    self.report.warning(traceback.format_exc(), preformatted=True)
            element = etree.fromstring(mathml)
            display_attrib = element.attrib["display"]
            lang_attrib = element.attrib[
                "{http://www.w3.org/XML/1998/namespace}lang"]
            if display_attrib == "inline":
                if lang_attrib == "nb" or lang_attrib == "nob" or lang_attrib == "nn":
                    return "<span>Matematisk formel</span>"
                else:
                    return "<span>Mathematical formula</span>"
            else:
                if lang_attrib == "nb" or lang_attrib == "nob" or lang_attrib == "nn":
                    return "<p>Matematisk formel</p>"
                else:
                    return "<p>Mathematical formula</p>"
Пример #30
0
    def email(self, recipients, subject=None, should_email=True, should_message_slack=True, should_attach_log=True, should_escape_chars=True):
        if not subject:
            assert isinstance(self.title, str) or self.pipeline is not None, "either title or pipeline must be specified when subject is missing"
            subject = self.title if self.title else self.pipeline.title

        smtp = {
            "host": Config.get("email.smtp.host", None),
            "port": Config.get("email.smtp.port", None),
            "user": Config.get("email.smtp.user", None),
            "pass": Config.get("email.smtp.pass", None)
        }
        sender = Address(Config.get("email.sender.name", "undefined"), addr_spec=Config.get("email.sender.address", "*****@*****.**"))

        # 0. Create attachment with complete log (including DEBUG statements)
        if should_attach_log is True:
            self.attachLog()

        attachments = []
        for m in self._messages["attachment"]:
            smb, file, unc = Filesystem.networkpath(m["text"])
            base_path = Filesystem.get_base_path(m["text"], self.pipeline.dir_base)
            relpath = os.path.relpath(m["text"], base_path) if base_path else None
            if m["text"].startswith(self.reportDir()):
                relpath = os.path.relpath(m["text"], self.reportDir())
            if not [a for a in attachments if a["unc"] == unc]:
                attachments.append({
                    "title": "{}{}".format(relpath, ("/" if os.path.isdir(m["text"]) else "")),
                    "smb": smb,
                    "file": file,
                    "unc": unc,
                    "severity": m["severity"]
                })

        # Determine overall status
        status = "INFO"
        for message_type in self._messages:
            for m in self._messages[message_type]:

                if m["severity"] == "SUCCESS" and status in ["INFO"]:
                    status = "SUCCESS"
                elif m["severity"] == "WARN" and status in ["INFO", "SUCCESS"]:
                    status = "WARN"
                elif m["severity"] == "ERROR":
                    status = "ERROR"

        try:
            assert isinstance(smtp, dict), "smtp must be a dict"
            assert isinstance(sender, Address), "sender must be a Address"
            assert isinstance(recipients, str) or isinstance(recipients, list) or isinstance(recipients, tuple), "recipients must be a str, list or tuple"
            assert isinstance(self.title, str) or self.pipeline and isinstance(self.pipeline.title, str), "title or pipeline.title must be a str"

            if isinstance(recipients, str):
                recipients = [recipients]
            elif isinstance(recipients, tuple):
                recipients = list(recipients)

            if status == "ERROR":
                for key in Config.get("administrators", default=[]):
                    if key not in recipients:
                        recipients.append(key)

            # when testing, only allow e-mail addresses defined in the ALLOWED_EMAIL_ADDRESSES_IN_TEST env var
            if Config.get("test"):
                subject = "[test] " + subject
                filtered_recipients = []
                for recipient in recipients:
                    if recipient in Config.get("email.allowed_email_addresses_in_test"):
                        filtered_recipients.append(recipient)
                recipients = filtered_recipients

            # 1. join lines with severity SUCCESS/INFO/WARN/ERROR
            markdown_text = []
            for m in self._messages["message"]:
                if should_escape_chars:
                    text = m['text'].replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
                else:
                    text = m['text']
                if m['preformatted'] is True:
                    markdown_text.append("<pre>{}</pre>".format(text))
                elif m['severity'] != 'DEBUG':
                    markdown_text.append(text)
            if attachments != [] or should_attach_log is True:
                markdown_text.append("\n----\n")
                markdown_text.append("\n# Lenker\n")
                markdown_text.append("\n<ul style=\"list-style: none;\">")

                # Pick icon and style for INFO-attachments
                attachment_styles = {
                    "DEBUG": {
                        "icon": "🗎",
                        "style": ""
                    },
                    "INFO": {
                        "icon": "🛈",
                        "style": ""
                    },
                    "SUCCESS": {
                        "icon": "😄",
                        "style": "background-color: #bfffbf;"
                    },
                    "WARN": {
                        "icon": "😟",
                        "style": "background-color: #ffffbf;"
                    },
                    "ERROR": {
                        "icon": "😭",
                        "style": "background-color: #ffbfbf;"
                    }
                }

                for attachment in attachments:
                    # UNC links seems to be preserved when viewed in Outlook.
                    # file: and smb: URIs are disallowed or removed.
                    # So these links will only work in Windows.
                    # If we need this to work cross-platform, we would have
                    # to map the network share paths to a web server so that
                    # the transfers go through http:. This could maybe be mapped
                    # using environment variables.
                    li = "<li>"
                    li += "<span style=\"vertical-align: middle; font-size: 200%;\">" + attachment_styles[attachment["severity"]]["icon"] + "</span> "
                    li += "<span style=\"vertical-align: middle; " + attachment_styles[attachment["severity"]]["style"] + "\">"
                    li += "<a href=\"file:///" + attachment["unc"] + "\">" + attachment["title"] + "</a> "
                    li += "<a href=\"" + attachment["smb"] + "\">" + self.img_string + "=\" alt=\"" + attachment["smb"] + "\"/>" + "</a> "
                    li += "</span>"
                    li += "</li>"
                    markdown_text.append(li)
                markdown_text.append("</ul>\n")
                label_string = ""
                for label in self.pipeline.labels:
                    label_string += "[{}] ".format(label)
                markdown_text.append("\n[{}] {} [{}] [status:{}]".format(self.pipeline.uid, label_string, self.pipeline.publication_format, status))
            markdown_text = "\n".join(markdown_text)

            # 2. parse string as Markdown and render as HTML
            if should_escape_chars:
                markdown_html = markdown.markdown(markdown_text, extensions=['markdown.extensions.fenced_code', 'markdown.extensions.codehilite'])
            else:
                markdown_html = markdown_text
            markdown_html = '''<!DOCTYPE html>
<html>
<head>
<meta charset=\"utf-8\"/>
<title>''' + subject.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;") + '''</title>
</head>
<body>
''' + markdown_html + '''
</body>
</html>
'''

            if not should_email:
                logging.info("[e-mail] Not sending email")
            else:
                # 3. build e-mail
                msg = EmailMessage()
                msg['Subject'] = re.sub(r"\s", " ", subject).strip()
                msg['From'] = sender
                msg['To'] = Report.emailStringsToAddresses(recipients)
                msg.set_content(markdown_text)
                msg.add_alternative(markdown_html, subtype="html")
                logging.info("[e-mail] E-mail with subject '{}' will be sent to: {}".format(msg['Subject'], ", ".join(recipients)))

                # 4. send e-mail
                if smtp["host"] and smtp["port"]:
                    smtp_server = "{}:{}".format(smtp["host"], smtp["port"])
                    logging.info("[e-mail] SMTP server: {}".format(smtp_server))
                    with smtplib.SMTP(smtp_server) as s:
                        s.ehlo()
                        # s.starttls()
                        if smtp["user"] and smtp["pass"]:
                            s.login(smtp["user"], smtp["pass"])
                        else:
                            logging.debug("[e-mail] user/pass not configured")
                        logging.debug("[e-mail] sending…")
                        s.send_message(msg)
                        logging.debug("[e-mail] sending complete.")
                else:
                    logging.warning("[e-mail] host/port not configured")

                temp_md_obj = tempfile.NamedTemporaryFile(suffix=".md")
                temp_html_obj = tempfile.NamedTemporaryFile(suffix=".html")
                with open(temp_md_obj.name, "w") as f:
                    f.write(markdown_text)
                    logging.debug("[e-mail] markdown: {}".format(temp_md_obj.name))
                with open(temp_html_obj.name, "w") as f:
                    f.write(markdown_html)
                    logging.debug("[e-mail] html: {}".format(temp_html_obj.name))
                if should_attach_log is True:
                    path_mail = os.path.join(self.reportDir(), "email.html")
                    shutil.copy(temp_html_obj.name, path_mail)
                    self.mailpath = Filesystem.networkpath(path_mail)
                else:
                    yesterday = datetime.now() - timedelta(1)
                    yesterday = str(yesterday.strftime("%Y-%m-%d"))
                    path_mail = os.path.join(self.pipeline.dir_reports, "logs", "dagsrapporter", yesterday, self.pipeline.uid + ".html")
                    shutil.copy(temp_html_obj.name, path_mail)
                    self.mailpath = Filesystem.networkpath(path_mail)

        except AssertionError as e:
            logging.error("[e-mail] " + str(e))
        if not should_message_slack:
            logging.warning("Not sending message to slack")
        else:
            # 5. send message to Slack
            slack_attachments = []
            for attachment in attachments:
                color = None
                if attachment["severity"] == "SUCCESS":
                    color = "good"
                elif attachment["severity"] == "WARN":
                    color = "warning"
                elif attachment["severity"] == "ERROR":
                    color = "danger"
                slack_attachments.append({
                    "title_link": attachment["smb"],
                    "title": attachment["title"],
                    "fallback": attachment["title"],
                    "color": color
                })
            Slack.slack(text=subject, attachments=slack_attachments)
Пример #31
0
    def emailPlainText(subject, message, recipients, should_email=True):
        assert isinstance(subject, str), "subject must be a str, was: {}".format(type(subject))
        assert isinstance(message, str), "message must be a str, was: {}".format(type(message))

        if recipients is None:
            logging.info("No recipients given, e-mail won't be sent: '" + subject + "'")
            return

        assert isinstance(recipients, str) or isinstance(recipients, list), (
            "recipients must be a str or list, was: {}".format(type(recipients))
        )

        smtp = {
            "host": Config.get("email.smtp.host", None),
            "port": Config.get("email.smtp.port", None),
            "user": Config.get("email.smtp.user", None),
            "pass": Config.get("email.smtp.pass", None)
        }
        sender = Address(Config.get("email.sender.name", "undefined"), addr_spec=Config.get("email.sender.address", "*****@*****.**"))

        if isinstance(recipients, str):
            recipients = [recipients]

        if not should_email:
            logging.info("[e-mail] Not sending plain text email")
        else:
            if Config.get("test"):
                subject = "[test] " + subject
                filtered_recipients = []
                for recipient in recipients:
                    if recipient in Config.get("email.allowed_email_addresses_in_test"):
                        filtered_recipients.append(recipient)
                recipients = filtered_recipients

            # 1. build e-mail
            msg = EmailMessage()
            msg['Subject'] = subject
            msg['From'] = sender
            msg['To'] = Report.emailStringsToAddresses(recipients)
            msg.set_content(message)

            # 2. send e-mail
            if not msg["To"]:
                logging.warning("[e-mail] Email with subject \"{}\" has no recipients".format(subject))
            else:
                logging.info("[e-mail] Sending email with subject \"{}\" to: {}".format(subject, ", ".join(recipients)))
                if isinstance(smtp["host"], str) and isinstance(smtp["port"], str):
                    with smtplib.SMTP(smtp["host"] + ":" + smtp["port"]) as s:
                        s.ehlo()
                        # s.starttls()
                        if smtp["user"] and smtp["pass"]:
                            s.login(smtp["user"], smtp["pass"])
                        else:
                            logging.debug("[e-mail] user/pass not configured")
                        logging.debug("[e-mail] sending…")
                        s.send_message(msg)
                        logging.debug("[e-mail] sending complete.")
                else:
                    logging.warning("[e-mail] host/port not configured")

        Slack.slack(text=subject, attachments=None)
Пример #32
0
            try:
                vxCage.exportVxCage(sha256, dumpDir)  
            except (Exception) as e:
                log.error("Export-Error: %s" % e)        
    elif args['which'] == 'export':
        log.info("Exporting {} {}".format(args['dirname'], args['sha256_file']))
        dumpDir = os.path.normpath(args['dirname']) + os.sep
        
        for sha256 in iterateSha256File(args['sha256_file'], args['json']):
            try:
                rapickerExport(sha256, dumpDir, database, vxCage)      
            except (Exception) as e:
                log.error("Export-Error: %s" % e)
    elif args['which'] == 'codeDBTag':
        log.info("codeDBTag {} {}".format(args['tag'], args['sha256_file']))
        config = cfgReporting.get("codeDB")

        if not config.get("enabled"):
            log.error("Sorry: CodeDB for Ragpicker is not enabled!")
            sys.exit()  
        
        for sha256 in iterateSha256File(args['sha256_file'], args['json']):
            try:
                submitTag(sha256, args['tag'], config)      
            except (Exception) as e:
                log.error("CodeDBTag-Error: %s" % e)
    elif args['which'] == 'import':
        log.info("Importing {}".format(args['dirname']))
        impDir = os.path.normpath(args['dirname']) + os.sep
        
        for file in os.listdir(impDir):
Пример #33
0
    def update_list_of_books(format, identifiers, report=logging):
        if format == "XHTML":  # "XHTML" means e-book (we're reusing identifiers previously reserved for XHTML files)

            catalog_changes_needed_filesize = 0
            catalog_changes_needed_formatklar = 0

            editions = Bibliofil.list_all_editions()
            filesize_xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
            filesize_xml += "<root>\n"
            lines_formatklar = []

            for book in editions:
                identifier = book["identifier"]
                library = book["library"]

                if identifier in identifiers:

                    if library is None or library.upper() != "NLB":
                        logging.info(
                            "book_available: only NLB books should have distribution methods: {} / {}"
                            .format(identifier, library))
                        continue

                    epub_dir = os.path.join(Directory.dirs_flat["epub-ebook"],
                                            book["identifier"])
                    has_epub = os.path.isdir(epub_dir)
                    if not has_epub:
                        continue
                    has_html = has_epub  # generated based on the EPUB on the fly
                    # has_docx = has_epub  # generated based on the EPUB on the fly
                    has_mobi = has_epub  # generated based on the EPUB on the fly

                    size = 0
                    if has_epub:
                        for root, dirs, files in os.walk(epub_dir):
                            for file in files:
                                size += os.path.getsize(
                                    os.path.join(root, file))
                        size_catalog = 0
                        if book["fileSize"] is not None:
                            size_catalog = int(book["fileSize"])
                        if size != size_catalog and size != 0:
                            logging.info(
                                f"Filesize for edition: {identifier} is not correct, will update. New filesize: {size} vs old {size_catalog}"
                            )
                            catalog_changes_needed_filesize += 1
                            filesize_xml += "<folder><name>{}</name><sizedata>{}</sizedata></folder>\n".format(
                                book["identifier"], size)

                    distribution_formats = Bibliofil.distribution_formats_epub(
                        has_epub, has_html, has_mobi, size)
                    distribution_formats_catalog = book["distribution"]

                    if distribution_formats != distribution_formats_catalog:

                        catalog_changes_needed_formatklar += 1
                        logging.info(
                            f"Distribution formats for edition {identifier} is not correct, will update. Distribution formats: {distribution_formats} vs old {distribution_formats_catalog}"
                        )

                        for distribution_format in distribution_formats:
                            lines_formatklar.append("{};{};{};{}".format(
                                identifier, distribution_format["name"],
                                distribution_format["format"],
                                distribution_format["method"]))

            if catalog_changes_needed_filesize > 0:
                logging.info(
                    f"{catalog_changes_needed_filesize} filesize catalog changes needed"
                )
                filesize_xml += "</root>\n"
                logging.info(
                    "Sending filesize-e-mail to {} with content:".format(
                        Config.get("email.filesize.address")))
                logging.info(filesize_xml)
                Report.emailPlainText("filesize: ", filesize_xml,
                                      Config.get("email.filesize.address"))

            if catalog_changes_needed_formatklar > 0:
                logging.info(
                    f"{catalog_changes_needed_formatklar} formatklar catalog changes needed"
                )
                logging.info("\n".join(lines_formatklar))
                Report.emailPlainText("formatklar: ",
                                      "\n".join(lines_formatklar),
                                      Config.get("email.formatklar.address"))
            if catalog_changes_needed_filesize == catalog_changes_needed_formatklar == 0:
                logging.info(f"No catalog changes needed for format {format}")

        else:
            report.debug("unknown format: {}".format(format))
            return
Пример #34
0
    def book_available(format, identifier, report=logging, title=""):
        library = None

        # get the appropriate book identifier(s)
        logging.info("{}/editions/{}?edition-metadata=all".format(
            Config.get("nlb_api_url"), identifier))
        response = requests.get("{}/editions/{}?edition-metadata=all".format(
            Config.get("nlb_api_url"), identifier))
        if response.status_code == 200:
            data = response.json()['data']
            library = data["library"]

        if format == "XHTML":  # "XHTML" means e-book (we're reusing identifiers previously reserved for XHTML files)
            lines = []

            epub_dir = os.path.join(Directory.dirs_flat["epub-ebook"],
                                    identifier)
            has_epub = os.path.isdir(epub_dir)
            has_html = has_epub  # generated based on the EPUB on the fly
            # has_docx = has_epub  # generated based on the EPUB on the fly
            has_mobi = has_epub  # generated based on the EPUB on the fly

            size = 0
            if has_epub:
                for root, dirs, files in os.walk(epub_dir):
                    for file in files:
                        size += os.path.getsize(os.path.join(root, file))

                filesize_xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
                filesize_xml += "<root>\n"
                filesize_xml += "<folder><name>{}</name><sizedata>{}</sizedata></folder>\n".format(
                    identifier, size)
                filesize_xml += "</root>\n"

                logging.info(
                    "Sending filesize-e-mail to {} with content:".format(
                        Config.get("email.filesize.address")))
                logging.info(filesize_xml)
                Report.emailPlainText("filesize: " + identifier, filesize_xml,
                                      Config.get("email.filesize.address"))

            distribution_formats = Bibliofil.distribution_formats_epub(
                has_epub, has_html, has_mobi, size)
            for distribution_format in distribution_formats:
                lines.append("{};{};{};{}".format(
                    identifier, distribution_format["name"],
                    distribution_format["format"],
                    distribution_format["method"]))

            if library is None or library.upper() != "NLB":
                report.debug(
                    "book_available: only NLB books should have distribution methods: {} / {}"
                    .format(identifier, library))
                lines = []

            logging.info(
                "Sending formatklar-e-mail to {} with content:".format(
                    Config.get("email.formatklar.address")))
            logging.info("\n".join(lines))
            Report.emailPlainText("formatklar: " + identifier,
                                  "\n".join(lines),
                                  Config.get("email.formatklar.address"))

        if format == "DAISY 2.02":
            lines = []
            text = "Abklar;624595;08.12.2021;624595112021;November;"
            #Abklar;nummer;dagensdato;boknummer-måned-år(mappenummer);Tittel(dc:title eller fra metadata for utgave)eks: Hjemmet, 44/2021
            #string emailMessage = "Abklar;" + tidskrift + ";" + todaydatemail + ";" + finalDirectory + ";" + myTitle+";";
            #Aviser
            #string emailMessage = "Abklar;" + folderNameExp + ";" + todaydatemail + ";" + folderNameExp + Program.folderDatePerodika+ ";"+  ";";
            dato = datetime.today().strftime('%d.%m.%Y')
            mmyyyy = datetime.today().strftime('%m%Y')
            identifier_short = identifier[:6]
            text = f"Abklar;{identifier_short};{dato};{identifier};{title};"
            logging.info("Sending abklar-e-mail to {} with content:".format(
                Config.get("email.abklar.address")))
            logging.info(text)
            Report.emailPlainText("Abklar", text,
                                  Config.get("email.abklar.address"))

        else:
            report.debug("book_available: unknown format: {}".format(format))
            return
Пример #35
0
import logging
from core.config import Config

config = Config()
config.parse()

level = config.get('log.level')

logging.basicConfig(level=level)
logger = logging
Пример #36
0
        os.makedirs(CACHEFOLDER)
    PIDLOG = "%sautohome.pid" % CACHEFOLDER
    Config.set("global", 'PID', PIDLOG)
    init_log()


def main():
    writePID(PIDLOG)
    from cmd.weather import Weather
    target = Weather('上海')
    target.get()


if __name__ == "__main__":
    init()
    PIDLOG = Config.get('global', 'PID')
    atexit.register(lambda: deletePID(PIDLOG))
    params = sys.argv
    if len(params) < 2:
        Usage()
    else:
        if params[1] == "start":
            if os.path.exists(PIDLOG):
                logging.info("Program is running")
                Usage()
            else:
                main()
        elif params[1] == "stop":
            stop(PIDLOG)
        elif params[1] == "restart":
            pass
Пример #37
0
 def get_all_users(self, callback=None):
     param = dict()
     param['maxResults'] = 100
     param['domain'] = Config.get('GOOGLE_DOMAIN')
     param['orderBy'] = 'familyName'
     self._get_all_users_inner(param, callback=callback)
Пример #38
0
 def create_service(self):
     self.service = discovery.build('admin', 'directory_v1', credentials=(
         ServiceAccountCredentials.from_json_keyfile_name(Config.get('CREDENTIALS_FILE_PATH')) \
             .create_scoped(Config.get('SCOPES')).create_delegated(Config.get('DELEGATED_USER'))
     ))
Пример #39
0
    PIDLOG = "%sautohome.pid" % CACHEFOLDER
    Config.set("global", "PID", PIDLOG)
    init_log()


def main():
    writePID(PIDLOG)
    from cmd.weather import Weather

    target = Weather("上海")
    target.get()


if __name__ == "__main__":
    init()
    PIDLOG = Config.get("global", "PID")
    atexit.register(lambda: deletePID(PIDLOG))
    params = sys.argv
    if len(params) < 2:
        Usage()
    else:
        if params[1] == "start":
            if os.path.exists(PIDLOG):
                logging.info("Program is running")
                Usage()
            else:
                main()
        elif params[1] == "stop":
            stop(PIDLOG)
        elif params[1] == "restart":
            pass