def create_zipped_application_packages(basedir): basedir = basedir.replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME) if os.path.exists(basedir): shutil.rmtree(basedir) disk.prepare_path(basedir) applications = [] apps_path = homing.home_join('storage/col/fileinspect_applications/') for path in os.listdir(apps_path): if os.path.isdir(os.path.join(apps_path, path)): applications.append(path) for dirname, subdirs, files in os.walk(apps_path): for f in files: if f.endswith(".pyc"): os.unlink(os.path.join(dirname, f)) for app in applications: outfilename = os.path.join(basedir, '%s.fi' % app) try: zf = zipfile.PyZipFile(outfilename, mode='w') zf.writepy(os.path.join(apps_path, app)) finally: zf.close() return
def __init__(self, conn, server, config, db_file): ftpserver.FTPHandler.__init__(self, conn, server) self.config = config = textual.utf8(config) self.ip = inet.get_ip(conn.getpeername()) self.db_file = db_file self.config_ip = config_reader.get_config_ip(self.ip, config) if not self.config_ip: conn.send( 'Please add your device %s to ftp_collector in LogInspect to send logs.\n' % self.ip) self.close() return self.profiles = config['client_map'][self.config_ip] # TODO use hashed password in config file self.authorizer = ftpserver.DummyAuthorizer() for user, profile in self.profiles.iteritems(): password = profile['password'] permission = profile['permission'] basedir = config['basedir'].replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME) home = profile['home'].lstrip('/') # let home not be absolute path user_home = os.path.join(basedir, home) disk.prepare_path(user_home + '/') self.authorizer.add_user(user, password, user_home, permission)
def __init__(self, conn, server, config, db_file, parser_name_only): ftpserver.FTPHandler.__init__(self, conn, server) self.config = config = textual.utf8(config) self.ip = inet.get_ip(conn.getpeername()) self.db_file = db_file self.parser_name_only = parser_name_only self.config_ip = config_reader.get_config_ip(self.ip, config) if not self.config_ip: conn.send("Please add your device %s to ftp_collector in LogInspect to send logs.\n" % self.ip) self.close() return self.profiles = config["client_map"][self.config_ip] self.authorizer = ftpserver.DummyAuthorizer() for user, profile in self.profiles.iteritems(): password = outself.get_decrypted_password(profile["password"]) permission = profile["permission"] basedir = config["basedir"].replace("$LOGINSPECT_HOME", homing.LOGINSPECT_HOME) home = profile["home"].lstrip("/") # let home not be absolute path user_home = os.path.join(basedir, home) disk.prepare_path(user_home + "/") self.authorizer.add_user(user, password, user_home, permission)
def main(): config = _parse_args() zmq_context = zmq.Context() db_file = homing.home_join('storage/col/filesystem_collector', 'checksums.shelve') disk.prepare_path(db_file) cursor_shelve = shelve.open(db_file, protocol=2) watcher.monitor(config, cursor_shelve, zmq_context)
def generate_certificate(ip, object_name, sic_one_timer_password, secret_key): #if all the cert files are present, no need to extract the certificates ip_path = os.path.join(storage_path, ip) if is_certificate_present(ip) == True: logging.warn("Cretificates already present.") return {'success': True, 'msg': 'Certificate already present'} #if not all the necessary files are present, remove all and proceed to generating fresh certificates elif not os.path.exists(ip_path): pass else: remove_certificate_if_exists(ip) #if the path to ip is not present it is created disk.prepare_path(ip_path + '/') os.chdir(ip_path) logging.warn("Creating Opsec Certificate...") sp = subprocess.Popen([os.path.join(certificate_path, 'opsec_pull_cert'), '-h', ip, '-n', object_name, '-p',\ sic_one_timer_password], stdout=subprocess.PIPE, stderr=subprocess.PIPE) sp.communicate() if not os.path.exists(os.path.join(storage_path, ip, 'opsec.p12')): return { 'success': False, 'msg': 'couldnt retrieve certificate from the server, RESET SIC in the server \ and try again' } logging.warn("Successfully Created Certificate : %r" % 'opsec.p12') logging.warn("Creating SSL Authorization Files...") sp = subprocess.Popen([os.path.join(certificate_path, 'opsec_putkey'), '-ssl', '-port', '18184', '-p',\ secret_key, ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE) sp.communicate() if not (os.path.exists(os.path.join(storage_path, ip, 'sslauthkeys.C')) or \ os.path.exists(os.path.join(storage_path, ip, 'sslsess.C'))): return { 'success': False, 'msg': 'couldnt retrieve ssl authorization certificates. Did you forget to do a \ fw putkey command for the LI on your server' } logging.warn("Successfully Created SSL Authorization Files : %r, %r" % ('sslauthkeys.C', 'sslsess.C')) logging.warn("Process Complete") return { 'success': True, 'msg': 'Certificate Sucessfully Created. Happy Opsecing!' }
def fetch_job(sid, config, db_file): log.debug('fetching files for sid:%s', sid) source = config['client_map'][sid] basedir = config['basedir'].replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME) ip = source['ip'] port = source['port'] user = source['user'] path = source['path'] password = source['password'] parser = source['parser'] charset = source['charset'] device_name = source['device_name'] localdir = os.path.join(basedir, ip, base64.urlsafe_b64encode(sid)) ftp = ftpclient.login(ip, port, user, password) for remotefile, mtime in ftpclient.fetch_file_mtime(ftp, path): disk.prepare_path(localdir + '/') vc = shelves.VersionChecker(db_file, sid, remotefile, mtime) if vc.is_older_version(): continue localfile = os.path.join(localdir, base64.urlsafe_b64encode(remotefile)) log.info('Downloading remote file %r to %r', remotefile, localfile) try: ftpclient.download(ftp, remotefile, localfile) except Exception, err: log.warn("fetching failed; remotefile=%s; sid=%s; error=%r", remotefile, sid, err) continue col_ts = time.time() cursor = vc.get_old_cursor(localfile) if cursor < 0: continue file_handler.main(sid, col_ts, parser, localfile, charset, device_name, source['normalizer'], source['repo'], cursor, source.get('regex_pattern'), source.get('regexparser_name'), conf_path=config.get('wiring_conf_path') or None)
def get_config_file_path(ip, opsec_sic_name, lea_server_opsec_entity_sic_name): #make a new lea.cong file and return its path #config_path = homing.home_join('storage/col/opsec_fetcher/%s/' % ip) config_path = '%s%s/' % (storage_path, ip) lea_conf_path = os.path.join(config_path, 'lea.conf') opsecp12_path = os.path.join(config_path, 'opsec.p12') disk.prepare_path(config_path) config_content = conf_template % ('ssl_opsec', ip, 18184, opsec_sic_name, opsecp12_path, lea_server_opsec_entity_sic_name) with open(lea_conf_path, 'w') as writer: writer.write(re.sub(pattern, '', config_content)) return lea_conf_path
def extract(filepath, db, config, updater): repo = config["repo"] repoinfo = get_repoinfo(db, repo) localdir = repoinfo.get("extracted_dir") if localdir: logging.warn("file already extracted at %r", localdir) return localdir logging.warn("extracting %r", filepath) updater.update_stat("extracting") cwd = os.path.abspath(os.curdir) destination = os.path.join(os.path.dirname(filepath), "extracted") disk.prepare_path(destination + '/') subtype = repoinfo["subtype"] if subtype in ["x-gzip"]: mode = 'r:gz' elif subtype in ["x-bzip2"]: mode = 'r:bz2' elif subtype in ["x-tar"]: mode = 'r:' else: raise Exception( "File %s has mime subtype %s which is not supported. Only 'x-gzip' and 'x-bzip2' supported" % (filepath, subtype)) try: tfile = tarfile.open(filepath, mode) tfile.extractall(destination) except: for key in ["downloaded_file", "subtype", "time"]: db.datatransport.update({"repo": repo}, {"$unset": {key: 1}}) db.datatransport.update({"repo": repo}, { "$set": { "status": "Uploaded file is corrupted. Please upload valid file." } }) os.unlink(filepath) shutil.rmtree(destination) raise db.datatransport.update({"repo": repo}, {"$set": { "extracted_dir": destination }}) os.chdir(cwd) updater.update_stat("extracted") return destination
def __init__(self, ip, object_name, sic_pwd, putkey_pwd): self.ip = ip self.object_name = object_name self.sic_pwd = sic_pwd self.putkey_pwd = putkey_pwd self.cert_path = os.path.join(STORAGE_PATH, ip) + "/" self.opsec_pull_cert = os.path.join(PULLERS_PATH, "opsec_pull_cert") self.opsec_putkey = os.path.join(PULLERS_PATH, "opsec_putkey") #make new dir for this ip to store its certificates disk.prepare_path(self.cert_path) #cd to the newly created dir to perform certificate actions os.chdir(self.cert_path)
def fetch_job(self): log.debug("fetching files for sid:%s", self.sid) config = self.fetcher_runner.get_config() try: source = config["client_map"][self.sid] except KeyError: log.debug("source for sid=%s has been deleted" % (self.sid)) return basedir = config["basedir"].replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME) scp_shelves_file = os.path.join(basedir, "scp.shelves") disk.prepare_path(scp_shelves_file) pd = pdict.PersistentDict(scp_shelves_file) if pd.get(self.sid): first_fetch = False else: first_fetch = True pd[self.sid] = True pd.sync() db_file = os.path.join(basedir, "checksums.pdict") remotepath = self.remotepath if remotepath.startswith('~'): remotepath = '.' + remotepath[1:] if '%' in self.device_ip: old_empty_dir = os.path.join(basedir, self.device_ip) if os.path.exists(old_empty_dir): try: shutil.rmtree(old_empty_dir) except: pass ip_dir = self.device_ip.replace("%", "_") else: ip_dir = self.device_ip localdir = os.path.join(basedir, ip_dir, base64.urlsafe_b64encode(self.sid)) try: password = self.get_decrypted_password(self.password) scp.setup(self.device_ip, self.port, self.user, password) except (ssh.SSHException, EOFError, SystemExit), err: log.warn("error while setting up connection; sid=%s", self.sid) return
def _get_conf_file_path(ip, opsec_sic_name, lea_server_opsec_entity_sic_name): #make a new lea.cong file and return its path config_path = homing.home_join('storage/col/opsec_fetcher/', ip, 'lea.conf') disk.prepare_path(config_path) with open(config_path, 'w') as conf_file: conf_file.write("lea_server auth_type ssl_opsec\n") conf_file.write("lea_server ip %s\n" % ip) conf_file.write("lea_server auth_port 18184\n") conf_file.write("opsec_sic_name %s\n" % opsec_sic_name) conf_file.write("opsec_sslca_file %s\n" % os.path.abspath("lea.conf")) conf_file.write("lea_server_opsec_entity_sic_name %s" % lea_server_opsec_entity_sic_name) return config_path
def initialize_memory(self, path_name): """ Call this function if memory feature for a fetcher is required. storage_path = storage/col/<path_name> sid_path = storage/col/path_name/<device_ip> mem_path = storage/col/path_name/<device_ip>/memory.mem These path will be created when memory initialization is done """ #Make path to the storage dir self.storage_path = os.path.join(COL_STORAGE_PATH, path_name) + "/" #Ensure path creation disk.prepare_path(self.storage_path) #Make path for this sid self.sid_path = os.path.join(self.storage_path, self.device_ip) #Make a class variable to use with the pd self.mem_path = os.path.join(self.sid_path, MEMORY_FILE) #Create memory.mem file at self.mem_path disk.prepare_path(self.mem_path)
def listen(db, config, updater): repo = config["repo"] localfile = get_downloaded_file(db, repo) if localfile: logging.warn("file already downloaded at %r", localfile) return localfile channel = config["upload_channel"] username = channel["username"] password = channel["password"] home = channel["home"].replace("$LOGINSPECT_HOME", homing.LOGINSPECT_HOME).replace("$repo", repo) disk.prepare_path(home + '/') address = ('0.0.0.0', channel["port"]) authorizer = ftpserver.DummyAuthorizer() authorizer.add_user(username, password, home, "elradfmwM") FTPHandler.authorizer = authorizer ftpd = ftpserver.FTPServer(address, lambda conn, server: FTPHandler(conn, server, db, repo)) logging.warn("ftp server starting at %r", address) ftpd.serve_forever() return DOWNLOADED_FILE
class SCPFetcher(Fetcher): def __init__(self, **args): super(SCPFetcher, self).__init__(**args) def fetch_job(self): log.debug("fetching files for sid:%s", self.sid) config = self.fetcher_runner.get_config() try: source = config["client_map"][self.sid] except KeyError: log.debug("source for sid=%s has been deleted" % (self.sid)) return basedir = config["basedir"].replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME) scp_shelves_file = os.path.join(basedir, "scp.shelves") disk.prepare_path(scp_shelves_file) pd = pdict.PersistentDict(scp_shelves_file) if pd.get(self.sid): first_fetch = False else: first_fetch = True pd[self.sid] = True pd.sync() db_file = os.path.join(basedir, "checksums.pdict") remotepath = self.remotepath if remotepath.startswith('~'): remotepath = '.' + remotepath[1:] if '%' in self.device_ip: old_empty_dir = os.path.join(basedir, self.device_ip) if os.path.exists(old_empty_dir): try: shutil.rmtree(old_empty_dir) except: pass ip_dir = self.device_ip.replace("%", "_") else: ip_dir = self.device_ip localdir = os.path.join(basedir, ip_dir, base64.urlsafe_b64encode(self.sid)) try: password = self.get_decrypted_password(self.password) scp.setup(self.device_ip, self.port, self.user, password) except (ssh.SSHException, EOFError, SystemExit), err: log.warn("error while setting up connection; sid=%s", self.sid) return try: for remotefile, mtime in scp.fetch_file_mtime( remotepath, self.name_pattern): disk.prepare_path(localdir + '/') if first_fetch: vc = shelves.VersionChecker(db_file, self.sid, remotefile, mtime=mtime, old_logs=self.old_logs) else: vc = shelves.VersionChecker(db_file, self.sid, remotefile, mtime=mtime) if vc.is_older_version(): continue localfile = os.path.join(localdir, base64.urlsafe_b64encode(remotefile)) log.info('Downloading remote file %r to %r', remotefile, localfile) try: scp.scp_get(remotefile, localfile) except (ssh.SSHException, EOFError, SystemExit), err: log.warn( "fetching failed; sid=%s; remotefile=%s; error=%r", self.sid, remotefile, err) continue col_ts = time.time() cursor = vc.get_old_cursor(localfile) if cursor < 0: continue conf_path = self.fetcher_runner.get_field_value_from_config( "wiring_conf_path") or None col_type = self.fetcher_runner.get_field_value_from_config( "col_type") client_map = self.get_client_map() file_handler.main(self.sid, col_type, col_ts, self.parser, localfile, self.charset, self.device_name, client_map["normalizer"], client_map["repo"], cursor, client_map.get("regex_pattern"), client_map.get("regexparser_name"), self.device_ip, conf_path=conf_path, source_name=remotefile) except gevent.GreenletExit: raise except (Exception, ssh.SSHException, EOFError, SystemExit), err: log.warn('exception while running job; sid=%s; err=%r', self.sid, err)
def create_shelve(db_file): disk.prepare_path(db_file) s = pdict.PersistentDict(db_file, 'c', format='pickle') return s
def _save_endings(endings, filename): from pylib import disk import cPickle as pickle disk.prepare_path(filename) pickle.dump(endings, open(filename, "wb"), pickle.HIGHEST_PROTOCOL)
pass ip_dir = ip.replace("%", "_") else: ip_dir = ip localdir = os.path.join(basedir, ip_dir, base64.urlsafe_b64encode(sid)) try: scp.setup(ip, port, user, password) except (ssh.SSHException, EOFError, SystemExit), err: log.warn('error while setting up connection; sid=%s', sid) return try: for remotefile, mtime in scp.fetch_file_mtime(remotepath): disk.prepare_path(localdir + '/') vc = shelves.VersionChecker(db_file, sid, remotefile, mtime=mtime) if vc.is_older_version(): continue localfile = os.path.join(localdir, base64.urlsafe_b64encode(remotefile)) log.info('Downloading remote file %r to %r', remotefile, localfile) try: scp.scp_get(remotefile, localfile) except (ssh.SSHException, EOFError, SystemExit), err: log.warn("fetching failed; sid=%s; remotefile=%s; error=%r", sid, remotefile, err) continue
def pre_create(self): id = is_mongokit_objectid(self.params.get("alert_id")) if id: alert = dboperation.read("AlertRules", {"_id": id}, True) if alert: notifications = alert.get("notification", []) if notifications: for notification in notifications: if notification.get("type") == "newtestemail": notifications.remove(notification) break if self.params.get("notify_newtestemail") == "on": email_template = textual.utf8( self.params.get('email_template')) try: template = Template(email_template) except TemplateSyntaxError: return ((0, 800), {}) email_emails = self.params.get('email_emails') if email_emails: email_emails = json.loads(email_emails) email_pattern = re.compile( r"^[-!#$%&'*+/0-9=?A-Z^_a-z{|}~](\.?[-!#$%&'*+/0-9=?A-Z^_a-z{|}~])*@[a-zA-Z](-?[a-zA-Z0-9])*(\.[a-zA-Z](-?[a-zA-Z0-9])*)*$" ) invalid_emails = [] for email in email_emails: if not bool(email_pattern.match(email)): invalid_emails.append(email) if invalid_emails: return ((0, 801), { "errors": { "invalid_emails": invalid_emails } }) else: return ((0, 801), {}) email_threshold_option = self.params.get( "email_threshold_option") email_threshold_value = self.params.get( "email_threshold_value") if email_threshold_value: email_threshold_value = int(email_threshold_value) template_file = "" if email_template: disk.prepare_path(ALERT_TEMPLATES_PATH) user_id = dboperation.read( "User", {'username': self.user.get_user_name()}, True) template_file = 'alert_%s_%s.tmp' % (str( user_id['_id']), base64.b32encode(alert["name"])) template_file_path = os.path.join( ALERT_TEMPLATES_PATH, template_file) email_template = Markup( email_template.decode('utf-8')).unescape() with open(template_file_path, 'w') as f: email_template = email_template.encode('utf-8') format_template = re.sub( '\|\s*(readable|date|time|datetime)\s*}}', self._regex_replacer, email_template) f.write(format_template) else: email_template = "<br>" disk.prepare_path(ALERT_TEMPLATES_PATH) template_file = 'alert_%s_%s.tmp' % ( self.user.get_user_name().encode( 'ascii', 'ignore'), base64.b32encode(name)) template_file_path = os.path.join( ALERT_TEMPLATES_PATH, template_file) email_template = Markup( email_template.decode('utf-8')).unescape() with open(template_file_path, 'w') as f: email_template = email_template.encode('utf-8') format_template = re.sub( '\|\s*(readable|date|time|datetime)\s*}}', self._regex_replacer, email_template) f.write(format_template) notifications.append({'template_file':template_file,'type':'newtestemail', 'notify_newtestemail':True, 'email_emails':email_emails,\ 'email_template':email_template, 'threshold':email_threshold_value, 'threshold_option':email_threshold_option}) return {"notification": notifications}