async def _login(client, token=None): if not token: try: log.info(f'{trace.cyan}> Attempting Login.') log.info( f'{trace.cyan}> Running on {trace.white}Discord{trace.green.s}Py ' f'{trace.cyan}v{trace.cyan.s}{discord.__version__}{trace.cyan}.' ) version.Discord.latest() version.YouTubeDL.latest() # token = json.json.reader('token') if token == enums.ReturnType.fail or token == enums.ReturnType.none: raise discord.errors.LoginFailure('No token') else: await client.start(crypt(token)) # client.run(crypt(token)) return except discord.errors.LoginFailure as e: if json.external.exists(json.default): try: os.remove(json.default) except OSError: pass log.critical( f'{type(e)} has occurred. Please check your login token') log.critical('SESSION HAS BEEN TERMINATED') log.critical(f'{e}') except Exception as err: # This should never occur. log.error(f'> {short_traceback()}') log.error(f'> {traceback.format_exc()}') else: await client.start(token) # client.run(token) return
def __init__(self, config): self.order = None self.subOrder = None self.orderItem = None self.data = None self.financeOrder = None self.test = None self.n_to_array = None self.data2 = None self.data3 = None self.full_outer_join = None self.testtest = None self.paymentFee = 0.02 self.marketFee = 0.01 self.platformFee = 0.05 self.config = config self.spark = SparkSession.builder.master(self.config.master).appName(self.config.app_name).getOrCreate() # .config(f'spark.mongodb.input.uri={self.config.input_mongodb_uri}') \ # .config(f'spark.mongodb.output.uri={self.config.output_mongodb_uri}') \ # .config('spark.driver.extraClassPath', self.config.jars_dir) \ # .getOrCreate() # self.spark.sparkContext.setLogLevel('INFO') log.info("=== Spark Info ===")
async def users(self, ctx): # LIST ALL USERS if not ctx.invoked_subcommand: log.info( f'Number of Users that this bot can see: {len(self.bot.users)}' ) for x in self.bot.users: log.info(f'{x.id}: {x}')
async def on_connect(self): if not self.bot.is_ready(): log.info(f'{trace.cyan}> Logging in at {time.readable.at()}.') else: log.debug( f'{trace.cyan}> Connection with {trace.white}Discord{trace.cyan} achieved.' )
def run(self, silent=False): if custom_help: self.client.remove_command('help') cog_count = 0 warnings = [] cogs = extensions() for extension in cogs: try: self.client.load_extension(extension) cog_count += 1 except Exception as e: if not silent: warnings.append(f'Failed to load extension {extension}\n{e}') if not silent: if not cogs: log.warn('No extensions were found.') else: for x in warnings: y = x.split('\n') log.warning(f'> {y[0]}') log.error(f'> {y[1]}') if len(warnings) > 0: # if saved() < enums.LogLevel.error.value: if len(warnings) == 1: log.error(f'> Failed to load {trace.yellow.s}{len(warnings)}{trace.cyan} extension.') else: log.error(f'> Failed to load {trace.yellow.s}{len(warnings)}{trace.cyan} extensions.') log.info(f'{trace.cyan}> Loaded {trace.yellow.s}{cog_count}{trace.cyan} extensions!')
def _save_embedded_files(out_dir, vm): """Save any extracted embedded files from the sample in the artifact directory. @param vm (ViperMonkey object) The ViperMonkey emulation engine object that did the emulation. @param out_dir (str) The artifact directory. """ # Make the output directory if needed. out_dir = safe_str_convert(out_dir) if (not os.path.exists(out_dir)): log.info("Making dropped sample directory ...") os.mkdir(out_dir) # Save each file. out_dir = safe_str_convert(out_dir) for file_info in vm.embedded_files: short_name = safe_str_convert(file_info[0]) long_name = safe_str_convert(file_info[1]) contents = safe_str_convert(file_info[2]) log.info("Saving embedded file " + long_name + " ...") try: f = open(out_dir + "/" + short_name, "w") f.write(contents) f.close() except IOError as e: log.error("Saving embedded file " + long_name + " failed. " + str(e))
def pull_embedded_pe_files(data, out_dir): """Directly pull out any PE files embedded in the given data. The PE files will be saved in a directory and will be named things like embedded*.exe. @param data (str) The contents of the file being analyzed. @param out_dir (str) The directory in which to save extracted PE files. """ # Is this a Office 2007 (zip) file? if core.filetype.is_office2007_file(data, is_data=True): # convert data to a BytesIO buffer so that we can use zipfile in memory # without writing a temp file on disk: data_io = io.BytesIO(data) # Pull embedded PE files from each file in the zip. with zipfile.ZipFile(data_io, "r") as f: for name in f.namelist(): curr_data = f.read(name) pull_embedded_pe_files(curr_data, out_dir) return # Is a PE file in the data at all? pe_pat = r"MZ.{70,80}This program (?:(?:cannot be run in DOS mode\.)|(?:must be run under Win32))" if (re.search(pe_pat, data) is None): return # There is an embedded PE. Break them out. # Get where each PE file starts. pe_starts = [] for match in re.finditer(pe_pat, data): pe_starts.append(match.span()[0]) pe_starts.append(len(data)) # Make the 2nd stage output directory if needed. if not os.path.isdir(out_dir): os.makedirs(out_dir) # Break out each PE file. Note that we probably will get extra data, # but due to the PE file format the file will be a valid PE (with an overlay). pos = 0 out_index = 0 while (pos < len(pe_starts) - 1): curr_data = data[pe_starts[pos]:pe_starts[pos+1]] curr_name = out_dir + "/embedded_pe" + safe_str_convert(out_index) + ".bin" # Make sure name is unique. while os.path.isfile(curr_name): out_index += 1 curr_name = out_dir + "/embedded_pe" + safe_str_convert(out_index) + ".bin" f = open(curr_name, "wb") f.write(curr_data) f.close() pos += 1 out_index += 1 log.info("Wrote embedded PE file to " + curr_name)
async def restart(self, ctx, *, message=None): if message is None: await ctx.send(f'{self.bot.user.name} is restarting.') else: await ctx.send(f'{self.bot.user.name} is restarting {message}.') log.info( f'{trace.red.s}> Manual Restart: {trace.yellow.s}{self.bot.user.name}, {trace.cyan.s}{self.bot.user.id}, {trace.magenta.s}Restarting.' ) await tls.Voice(ctx).disconnect() await self.bot.close()
def __init__(self, env): self.env = env self.file_dir = os.path.dirname(os.path.realpath('__file__')) self.file_path = '/config/%s-conf.json' % self.env self.full_path = self.file_dir + self.file_path log.info(f'Reading config from: {self.full_path}') with open(self.full_path) as config_file: self.data = json.load(config_file)
async def commands(self, ctx): com = [] for x in self.bot.walk_commands(): com.append(x) pos = len(com) com = remove_duplicates(com) for x in com: log.debug(x) # log(type(x)) log.info(f'{len(com)} commands (including sub-commands)') log.info(f'{pos} possible command combinations (including aliases)')
def song(input_song, output_song, folder, avconv=False): """ Do the audio format conversion. """ if not input_song == output_song: log.info('Converting {0} to {1}'.format(input_song, output_song.split('.')[-1])) if avconv: exit_code = convert_with_avconv(input_song, output_song, folder) else: exit_code = convert_with_ffmpeg(input_song, output_song, folder) return exit_code return 0
async def quit(self, ctx): for x in self.bot.commands: if x.name not in exceptions: x.enabled = False await tls.Voice(ctx).disconnect() global lockdown lockdown = True log.info( f'{trace.red.s}> Lockdown: {trace.yellow.s}{self.bot.user.name}, {trace.cyan.s}{self.bot.user.id}, {trace.magenta.s}Halted.' ) await self.bot.change_presence(status=discord.Status.do_not_disturb) await ctx.send(f'{self.bot.user.name} is now in lockdown.')
def __reg_consumer(self, update): msg = update.get('message') from_user = msg.get('from') if msg else None id = from_user.get('id') if from_user else None if not id: return f_name = from_user.get('first_name', '') l_name = from_user.get('last_name', '') u_name = from_user.get('username', '') log.info('[x] USER: {0} {1} {2} aka {3}'.format( id, f_name, l_name, u_name))
def input_link(links): """ Let the user input a choice. """ while True: try: log.info('Choose your number:') the_chosen_one = int(input('> ')) if 1 <= the_chosen_one <= len(links): return links[the_chosen_one - 1] elif the_chosen_one == 0: return None else: log.warning('Choose a valid number!') except ValueError: log.warning('Choose a valid number!')
def embed(music_file, meta_tags): """ Embed metadata. """ if meta_tags is None: log.warning('Could not find metadata') return None elif music_file.endswith('.m4a'): log.info('Applying metadata') return embed_m4a(music_file, meta_tags) elif music_file.endswith('.mp3'): log.info('Applying metadata') return embed_mp3(music_file, meta_tags) else: log.warning('Cannot embed metadata into given output extension') return False
async def invite(self, ctx, guild_id): # LIST ALL USERS guild = await self.bot.fetch_guild(guild_id) log.info(f'Getting invite for guild: {guild.id}: {guild}') channels = await guild.fetch_channels() invite = None for x in channels: if x.position == 0: if type(x).__name__ == 'TextChannel': invite = await x.create_invite( reason='Requested by bot owner.', unique=False, temporary=False, max_age=3600, max_uses=1) break log.info(f'Invite for guild {guild.id}: {invite}')
def getgcov(version): """Executes gcov script and returns its output version -- version of VLC to get gcov statistics return -- gcov output for version""" log.info("Collecting gcov data for {}".format(version)) devnull = open(os.devnull, 'wb') # stdout & stderr are redirected here) proc = subprocess.Popen(['./core/gcov.sh', config.downloaddir + version], stdout=devnull, stderr=devnull) proc.wait() fp = open("." + version + ".txt", 'r') output = fp.read() fp.close() return output
def build_vlc(version): """Build the vlc from sources (if needed)""" stampfile = config.downloaddir + "." + version vlcdir = os.getcwd() + "/" + config.downloaddir + version + "/" if os.path.exists(stampfile): log.info("No need to build {}".format(version)) return subprocess.check_call(["./configure", "--prefix=/usr", "--enable-coverage", "--disable-dbus", "--disable-lua", "--disable-mad", "--disable-postproc", "--disable-a52", "--disable-fribidi", "--enable-pulse", "--enable-alsa"], cwd=vlcdir) subprocess.check_call(["make"], cwd=vlcdir) open(stampfile, 'w').close()
def login(self): log.info(f'{trace.cyan}> Attempting Login.') log.info( f'{trace.cyan}> Running on {trace.white}Discord{trace.green.s}Py ' f'{trace.cyan}v{trace.cyan.s}{discord.__version__}{trace.cyan}.') version.Discord.latest() version.YouTubeDL.latest() tokens = json.json.orm['tokens']['discord'] threads = [ threading.Thread(target=login_threads, args=( self.prefix, crypt(x), ), daemon=True) for x in tokens ] # threads = [threading.Thread(target=login_threads, args=(self.prefix, crypt(x),)) for x in tokens] return threads
async def user(self, ctx, *, arg): # LIST ALL USERS user = await commands.UserConverter().convert(ctx=ctx, argument=arg) log.info(f'Viewing user: {user.id}: {user}') log.info(f'Joined: {user.created_at}') guilds = [] for x in self.bot.guilds: if user in x.members: guilds.append(x) log.info(f'Number of Guilds that {user.id} is in: {len(guilds)}') for x in guilds: log.info(f'{x.id}: {x.name}')
def form_valid(self, form, data): # Сущесвующие в базе дни days_exists = list( Day.objects.filter( date__year=form.cleaned_data['year'], ).values_list('date', flat=True)) # Дни для добавления days_raw = filter( lambda x: x['date'] not in days_exists, data, ) log.info("Starting import CSV...") errors = False for day_raw in days_raw: # Добавляем дни, используя форму day_form = DayForm(day_raw) if day_form.is_valid(): day_form.save() else: errors = True log.error("{}: {} ".format( model_to_dict(day_form.instance), day_form.errors.as_data(), )) if errors: log.error("CSV import finished with errors") messages.error( self.request, _("CSV import finished with errors (see more in logs)"), ) else: log.info("CSV import finished without errors") messages.success( self.request, _("CSV import finished without errors"), ) return super().form_valid(form)
def get_videofile(): """Downlaod zipped video, extract it to the download folder return -- path to video file""" if os.path.exists(config.downloaddir + os.path.basename(config.videourl).replace('exe', 'wmv')): log.info("Media file already downloaded and extracted") return log.info("Downloading media file") log.debug(config.videourl) videoexe, _ = urllib.urlretrieve(config.videourl) log.info("Extracting video from ZIP (sic!)") log.debug(videoexe) archfile = zipfile.ZipFile(videoexe) for f in archfile.namelist(): mediafile = open(f, "wb") mediafile.write(archfile.read(f)) mediafile.close() path = config.downloaddir + mediafile.name os.rename(mediafile.name, path) archfile.close() return path
async def dev(self, ctx): if not ctx.invoked_subcommand: com = [] for x in self.bot.walk_commands(): com.append(x) pos = len(com) com = remove_duplicates(com) log.info(f'{len(com)} commands (including sub-commands)') log.info( f'{pos} possible command combinations (including aliases)') t = tls.search(ctx.command.name, com) if len(t) > 0: log.info(f'Sub-commands under this command ({len(t)}):') for x in t: log.info(x)
def _remove_duplicate_iocs(iocs): """Remove IOC strings that are substrings of other IOC strings. @param iocs (list) List of IOCs (str). @return (set) The original IOC list with duplicate-ish IOC strings stripped out. """ # Track whether to keep an IOC string. r = set() skip = set() log.info("Found " + safe_str_convert(len(iocs)) + " possible IOCs. Stripping duplicates...") for ioc1 in iocs: # Does this IOC look like straight up garbage? if (read_ole_fields.is_garbage_vba(ioc1, test_all=True, bad_pct=.25)): skip.add(ioc1) continue # Looks somewhat sensible. See if it is a duplicate. keep_curr = True for ioc2 in iocs: if (ioc2 in skip): continue if ((ioc1 != ioc2) and (ioc1 in ioc2)): keep_curr = False break if ((ioc1 != ioc2) and (ioc2 in ioc1)): skip.add(ioc2) if (keep_curr): r.add(ioc1) # Return stripped IOC set. return r
def run(self, options=None): """ This method is the main function in this class. Pre: - Previous execution of detect_mode() method. Return: None """ if self.initial_backup_mode: log.info("INITIAL BACKUP MODE") log.debug("0 - READY BACKUP") self.prepare_to_init_backup() log.debug("2 - GENERATE ACTUAL FS") self.actual_filesystem.generate() log.debug("5.5 - UPLOAD ALL LOCAL FS") self.upload_all() log.debug("6 - UPDATE REMOTE FS") self.upload_actual_fs_struct() elif self.remote_home_mode: log.info("REMOTE_HOME MODE") log.debug("1 - LOAD REMOTE FS") self.get_remote_fs_struct() log.debug("2 - SYNC REMOTE HOME") self.sync_remote_home() elif self.resync_mode: # Reprocess log.info("RESYNC") log.debug("1 - LOAD REMOTE FS") self.get_remote_fs_struct() log.debug("2 - GENERATE ACTUAL FS") self.actual_filesystem.generate() log.debug("3,4 - CALCULATE CHANGES") changes = filesystem.compare_fs(actual_fs=self.actual_filesystem, old_fs=self.remote_filesystem) log.debug("5 - APPLY DIFERENCES (DELETE/DOWNLOAD AND UPLOAD)") self.process_changes_in_remote(changes) log.debug("6 - UPDATE REMOTE FS") self.upload_actual_fs_struct() else: log.critical("UNKNOWN MODE, existing...")
def get_vlc(version): """Download VLC sources from Internet and extract them version -- version of VLC to work with e.g. 2.0.4 return -- path to VLC binary""" if os.path.exists(config.downloaddir + version): log.info("VLC {} already downloaded and extracted".format(version)) return log.info("Downloading VLC version {} ".format(version)) url = config.vlan['url'].format(version[4:], version) log.debug(url) archvlc, _ = urllib.urlretrieve(url) log.info("Extracting VLC sources {} ".format(archvlc)) subprocess.check_call(['tar', '-C', config.downloaddir, '-Jxf', archvlc]) return config.downloaddir + version + "/vlc"
async def guild(self, ctx, guild_id): # LIST GUILD INFO guild = await self.bot.fetch_guild(guild_id) log.info(f'Viewing guild: {guild.id}: {guild}') owner = await commands.UserConverter().convert(ctx=ctx, argument=str( guild.owner_id)) log.info(f'Owner: {guild.owner_id}: {owner}') # log(f'Members: {len(guild.members)} members') # log(f'Categories: {len(guild.categories)} categories') # log(f'Channels: {len(guild.channels)} channels') log.info(f'Roles: {len(guild.roles)} roles') log.info(f'Emojis: {len(guild.emojis)} emojis') log.info(f'Verification Level: {guild.verification_level}') log.info(f'Content Filter: {guild.explicit_content_filter}') log.info(f'Notifications: {guild.default_notifications}') log.info(f'Voice Region: {guild.region}') if len(guild.features) > 0: log.info(f'Features: {len(guild.features)}') for x in guild.features: log.info(f'Feature: {x}') log.info(f'Created at: {guild.created_at}')
def init(): init_dependencies() log.info("=== Singularity ===") args = init_args_parser() log.info(f'Current environment: {args.env}') config = Configuration(args.env) log.info("=== Spark project is running... ===") log.info("== Configurations ==") log.info(f'app_name: {config.app_name}') log.info(f'input_mongodb_uri: {config.input_mongodb_uri}') log.info(f'output_mongodb_uri: {config.output_mongodb_uri}') log.info(f'jars_dir: {config.jars_dir}') log.info(f'master: {config.master}') sp = SparkManager(config) sp.run()
def init_dependencies(): log.info("=== Initial libs and modules ===") if os.path.exists('config.zip'): log.info("=== Found config.zip ===") sys.path.insert(0, 'config.zip') log.info("=== Added config.zip to path ===") else: sys.path.insert(0, './config') log.info("=== Added ./config to path ===") if os.path.exists('core.zip'): log.info("=== Found core.zip ===") sys.path.insert(0, 'core.zip') log.info("=== Added core.zip to path ===") else: sys.path.insert(0, './core') log.info("=== Added ./core to path ===") if os.path.exists('jars.zip'): log.info("=== Found jars.zip ===") sys.path.insert(0, 'jars.zip') log.info("=== Added jars.zip to path ===") else: sys.path.insert(0, './jars') log.info("=== Added ./jars to path ===") if os.path.exists('jobs.zip'): log.info("=== Found jobs.zip ===") sys.path.insert(0, 'jobs.zip') log.info("=== Added jobs.zip to path ===") else: sys.path.insert(0, './jobs') log.info("=== Added ./jobs to path ===") if os.path.exists('libs.zip'): log.info("=== Found libs.zip ===") sys.path.insert(0, 'libs.zip') log.info("=== Added libs.zip to path ===") else: sys.path.insert(0, './libs') log.info("=== Added ./libs to path ===") if os.path.exists('log.zip'): log.info("=== Found log.zip ===") sys.path.insert(0, 'log.zip') log.info("=== Added log.zip to path ===") else: sys.path.insert(0, './log') log.info("=== Added ./log to path ===")
async def guilds(self, ctx): # LIST ALL GUILDS log.info( f'Number of Guilds that this bot is in: {len(self.bot.guilds)}') for x in self.bot.guilds: log.info(f'{x.id}: {x.name}')
def _process_file (filename, data, strip_useless=False, entry_points=None, time_limit=None, display_int_iocs=False, artifact_dir=None, out_file_name=None, do_jit=False): """Process a single file. @param container (str) Path and filename of container if the file is within a zip archive, None otherwise. @param filename (str) path and filename of file on disk, or within the container. @param data (bytes) content of the file if it is in a container, None if it is a file on disk. @param strip_useless (boolean) Flag turning on/off modification of VB code prior to parsing. @param entry_points (list) A list of the names (str) of the VB functions from which to start emulation. @param time_limit (int) The emulation time limit, in minutes. If None there is not time limit. @param display_int_iocs (boolean) Flag turning on/off the reporting of intermediate IOCs (base64 strings and URLs) found during the emulation process. @param artifact_dir (str) The directory in which to save artifacts dropped by the sample under analysis. If None the artifact @param out_file_name (str) The name of the file in which to store the ViperMonkey analysis results as JSON. If None no JSON results will be saved. @param do_jit (str) A flag turning on/off doing VB -> Python transpiling of loops to speed up loop emulation. @return (list) A list of actions if actions found, an empty list if no actions found, and None if there was an error. """ # Increase Python call depth. sys.setrecursionlimit(13000) # Set the emulation time limit. if (time_limit is not None): core.vba_object.max_emulation_time = datetime.now() + timedelta(minutes=time_limit) # Create the emulator. log.info("Starting emulation...") vm = core.ViperMonkey(filename, data, do_jit=do_jit) orig_filename = filename if (entry_points is not None): for entry_point in entry_points: vm.user_entry_points.append(entry_point) try: #TODO: handle olefile errors, when an OLE file is malformed if (isinstance(data, Exception)): data = None vba = None try: vba = _get_vba_parser(data) except FileOpenError as e: # Is this an unrecognized format? if ("Failed to open file is not a supported file type, cannot extract VBA Macros." not in safe_str_convert(e)): # No, it is some other problem. Pass on the exception. raise e # This may be VBScript with some null characters. Remove those and try again. data = data.replace("\x00", "") vba = _get_vba_parser(data) # Do we have analyzable VBA/VBScript? Do the analysis even # without VBA/VBScript if we are scraping for intermediate # IOCs. if (vba.detect_vba_macros() or display_int_iocs): # Read in document metadata. try: log.info("Reading document metadata...") ole = olefile.OleFileIO(data) vm.set_metadata(ole.get_metadata()) except Exception as e: log.warning("Reading in metadata failed. Trying fallback. " + safe_str_convert(e)) vm.set_metadata(get_metadata_exif(orig_filename)) # If this is an Excel spreadsheet, read it in. vm.loaded_excel = excel.load_excel(data) # Set where to store directly dropped files if needed. if (artifact_dir is None): artifact_dir = "./" if ((filename is not None) and ("/" in filename)): artifact_dir = filename[:filename.rindex("/")] only_filename = filename if ((filename is not None) and ("/" in filename)): only_filename = filename[filename.rindex("/")+1:] # Set the output directory in which to put dumped files generated by # the macros. out_dir = None if (only_filename is not None): out_dir = artifact_dir + "/" + only_filename + "_artifacts/" if os.path.exists(out_dir): shutil.rmtree(out_dir) else: out_dir = "/tmp/tmp_file_" + safe_str_convert(random.randrange(0, 10000000000)) log.info("Saving dropped analysis artifacts in " + out_dir) core.vba_context.out_dir = out_dir del filename # We already have this in memory, we don't need to read it again. # Parse the VBA streams. log.info("Parsing VB...") comp_modules = parse_streams(vba, strip_useless) if (comp_modules is None): return None got_code = False for module_info in comp_modules: m = module_info[0] stream = module_info[1] if (m != "empty"): vm.add_compiled_module(m, stream) got_code = True if ((not got_code) and (not display_int_iocs)): log.info("No VBA or VBScript found. Exiting.") return ([], [], [], []) # Get the VBA code. vba_code = "" for (_, _, _, macro_code) in vba.extract_macros(): if (macro_code is not None): vba_code += macro_code # Do not analyze the file if the VBA looks like garbage. if (read_ole_fields.is_garbage_vba(vba_code)): raise ValueError("VBA looks corrupted. Not analyzing.") # Read in text values from all of the various places in # Office 97/2000+ that text values can be hidden. So many # places. read_ole_fields.read_payload_hiding_places(data, orig_filename, vm, vba_code, vba) # Do Emulation. safe_print("") safe_print('-'*79) safe_print('TRACING VBA CODE (entrypoint = Auto*):') if (entry_points is not None): log.info("Starting emulation from function(s) " + safe_str_convert(entry_points)) pyparsing.ParserElement.resetCache() vm.vba = vba vm.trace() # Done with emulation. # Report the results. str_actions, tmp_iocs, shellcode_bytes = _report_analysis_results(vm, data, display_int_iocs, orig_filename, out_file_name) # Save any embedded files as artifacts. _save_embedded_files(out_dir, vm) # Return the results. return (str_actions, vm.external_funcs, tmp_iocs, shellcode_bytes) # No VBA/VBScript found? else: safe_print('Finished analyzing ' + safe_str_convert(orig_filename) + " .\n") safe_print('No VBA macros found.') safe_print('') return ([], [], [], []) # Handle uncaught exceptions triggered during analysis. except Exception as e: # Print error info. if (("SystemExit" not in safe_str_convert(e)) and (". Aborting analysis." not in safe_str_convert(e))): traceback.print_exc() log.error(safe_str_convert(e)) # If this is an out of memory error terminate the process with an # error code indicating that there are memory problems. This is so # that higer level systems using ViperMonkey can see that there is a # memory issue and handle it accordingly. if isinstance(e, MemoryError): log.error("Exiting ViperMonkey with error code 137 (out of memory)") sys.exit(137) # Done. Analysis failed. return None
async def find(self, ctx, *, user): matches = tls.search(user, self.bot.users) log.info(f'Found {len(matches)} users that match "{user}"') for x in matches: log.info(f'{x.id}: {x.name}#{x.discriminator}')
def run(self): log.info("=== Spark exec now ===") self.order = self.spark.read \ .format("com.mongodb.spark.sql.DefaultSource") \ .option("collection", "orders") \ .load() self.order = self.order.alias('order') # order.limit(1).toPandas() self.order.show() log.info(Util.getShowString(self.order)) self.subOrder = self.spark.read \ .format("com.mongodb.spark.sql.DefaultSource") \ .option("collection", "orders.suborders") \ .load() self.subOrder = self.subOrder.alias('subOrder') # subOrder.select("*").toPandas() self.subOrder.show() log.info(self.subOrder.show()) self.orderItem = self.spark.read \ .format("com.mongodb.spark.sql.DefaultSource") \ .option("collection", "orders.items") \ .load() # orderItem = orderItem.alias('orderItem') self.orderItem.limit(1).toPandas() log.info(self.orderItem.limit(1).toPandas()) # orderItem.show() self.data = self.order.join(self.subOrder, self.order._id == self.subOrder.orderId) \ .join(self.orderItem, self.subOrder._id == self.orderItem.subOrderId) \ .filter(self.order._id == "hKhl3HlwNMQ") self.data.select(["subOrderNo", "productId", "productName", "productPrice", "totalQuantity"]).toPandas() self.data = self.subOrder.join(self.orderItem, self.subOrder._id == self.orderItem.subOrderId) \ .filter(self.subOrder.merchantId == "MEV747qhJ2f") self.data.select(["merchantId", "productName", "productPrice", "totalQuantity"]).toPandas() self.financeOrder = self.spark.read \ .format("com.mongodb.spark.sql.DefaultSource") \ .option("collection", "order.finance") \ .load() # financeOrder = subOrder.alias('subOrder') # financeOrder.select("*").toPandas() self.financeOrder = self.financeOrder.select(["code", "financeType", "action"]) self.financeOrder.show() log.info(self.financeOrder.show()) # financeOrder.printSchema() # financeOrder = financeOrder.groupBy('action').agg(collect_list('code').alias('code')) # testfinanceOrder = financeOrder.select("code") # print(testfinanceOrder) # testfinanceOrder.show() self.test = self.data.select(["merchantName", "totalQuantity", "productPrice"]) self.test = self.test.withColumn("productItem", lit(1)) self.test = self.test.withColumn("action", lit("order")) self.test.show() log.info(self.test.show()) log.info(self.test.show()) self.n_to_array = udf(lambda n: [n] * n, ArrayType(IntegerType())) self.data2 = self.test.withColumn('totalQuantity', self.n_to_array('totalQuantity')) # data2.show() self.data3 = self.data2.withColumn('totalQuantity', explode('totalQuantity')) # # data3.printSchema() # data3 = data3.select("*") self.data3.show() log.info(self.data3.show()) # ppp = financeOrder.crossJoin(data3) # # ppp = data3.crossJoin(financeOrder) # ppp.orderBy('merchantName', ascending=False).show() # df = df1.join(df2, on=['key'], how='inner') # df.show() # daaa = data3.join(financeOrder, data3.action == financeOrder.action).select("*") # # daaa.printSchema() # daaa.show(1) self.full_outer_join = self.data3.join(self.financeOrder, self.data3.action == self.financeOrder.action, how='full') # Could also use 'full_outer' self.full_outer_join.select(["merchantName", "productPrice", "productItem", "code", "financeType"]).show() self.testtest = self.full_outer_join.withColumn('Amount', when((col("financeType") == "paymentFee"), round((col('productPrice') * self.paymentFee), 2)) .when((col("financeType") == "marketFee"), round((col('productPrice') * self.marketFee), 2)) .when((col("financeType") == "platformFee"), round((col('productPrice') * self.platformFee), 2)) .otherwise(col('productPrice')) ) self.testtest.show() log.info(self.testtest.show())
def main(): """Main function, called when vipermonkey is run from the command line. """ # Increase recursion stack depth. sys.setrecursionlimit(13000) # print banner with version # Generated with http://www.patorjk.com/software/taag/#p=display&f=Slant&t=ViperMonkey safe_print(''' _ ___ __ ___ __ | | / (_)___ ___ _____/ |/ /___ ____ / /_____ __ __ | | / / / __ \/ _ \/ ___/ /|_/ / __ \/ __ \/ //_/ _ \/ / / / | |/ / / /_/ / __/ / / / / / /_/ / / / / ,< / __/ /_/ / |___/_/ .___/\___/_/ /_/ /_/\____/_/ /_/_/|_|\___/\__, / /_/ /____/ ''') safe_print('vmonkey %s - https://github.com/decalage2/ViperMonkey' % __version__) safe_print('THIS IS WORK IN PROGRESS - Check updates regularly!') safe_print('Please report any issue at https://github.com/decalage2/ViperMonkey/issues') safe_print('') DEFAULT_LOG_LEVEL = "info" # Default log level LOG_LEVELS = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL } usage = 'usage: %prog [options] <filename> [filename2 ...]' parser = optparse.OptionParser(usage=usage) parser.add_option("-r", action="store_true", dest="recursive", help='find files recursively in subdirectories.') parser.add_option("-z", "--zip", dest='zip_password', type='str', default=None, help='if the file is a zip archive, open first file from it, using the ' 'provided password (requires Python 2.6+)') parser.add_option("-f", "--zipfname", dest='zip_fname', type='str', default='*', help='if the file is a zip archive, file(s) to be opened within the zip. ' 'Wildcards * and ? are supported. (default:*)') parser.add_option("-e", action="store_true", dest="scan_expressions", help='Extract and evaluate/deobfuscate constant expressions') parser.add_option('-l', '--loglevel', dest="loglevel", action="store", default=DEFAULT_LOG_LEVEL, help="logging level debug/info/warning/error/critical (default=%default)") parser.add_option("-s", '--strip', action="store_true", dest="strip_useless_code", help='Strip useless VB code from macros prior to parsing.') parser.add_option("-j", '--jit', action="store_true", dest="do_jit", help='Speed up emulation by JIT compilation of VB loops to Python.') parser.add_option('-i', '--init', dest="entry_points", action="store", default=None, help="Emulate starting at the given function name(s). Use comma seperated " "list for multiple entries.") parser.add_option('-t', '--time-limit', dest="time_limit", action="store", default=None, type='int', help="Time limit (in minutes) for emulation.") parser.add_option("-c", '--iocs', action="store_true", dest="display_int_iocs", help='Display potential IOCs stored in intermediate VBA variables ' 'assigned during emulation (URLs and base64).') parser.add_option("-v", '--version', action="store_true", dest="print_version", help='Print version information of packages used by ViperMonkey.') parser.add_option("-o", "--out-file", action="store", default=None, type="str", help="JSON output file containing resulting IOCs, builtins, and actions") parser.add_option("-p", "--tee-log", action="store_true", default=False, help="output also to a file in addition to standard out") parser.add_option("-b", "--tee-bytes", action="store", default=0, type="int", help="number of bytes to limit the tee'd log to") (options, args) = parser.parse_args() # Print version information and exit? if (options.print_version): print_version() sys.exit(0) # Print help if no arguments are passed if len(args) == 0: safe_print(__doc__) parser.print_help() sys.exit(0) # setup logging to the console # logging.basicConfig(level=LOG_LEVELS[options.loglevel], format='%(levelname)-8s %(message)s') colorlog.basicConfig(level=LOG_LEVELS[options.loglevel], format='%(log_color)s%(levelname)-8s %(message)s') json_results = [] for container, filename, data in xglob.iter_files(args, recursive=options.recursive, zip_password=options.zip_password, zip_fname=options.zip_fname): # ignore directory names stored in zip files: if container and filename.endswith('/'): continue if options.scan_expressions: process_file_scanexpr(container, filename, data) else: entry_points = None if (options.entry_points is not None): entry_points = options.entry_points.split(",") process_file(container, filename, data, strip_useless=options.strip_useless_code, entry_points=entry_points, time_limit=options.time_limit, display_int_iocs=options.display_int_iocs, tee_log=options.tee_log, tee_bytes=options.tee_bytes, out_file_name=options.out_file, do_jit=options.do_jit) # add json results to list if (options.out_file): with open(options.out_file, 'r') as json_file: try: json_results.append(json.loads(json_file.read())) except ValueError: pass if (options.out_file): with open(options.out_file, 'w') as json_file: if (len(json_results) > 1): json_file.write(json.dumps(json_results, indent=2)) else: json_file.write(json.dumps(json_results[0], indent=2)) log.info("Saved results JSON to output file " + options.out_file)