def Parse(self, filename, file_object, configuration): """Parses a SQLite database file entry. Args: filename (str): name of the file. file_object (dfvfs.FileIO): file-like object. configuration: configuration values. Raises: UnableToParseFile: when the file cannot be parsed. """ temporary_directory_path = configuration.root_tmp_path + os.sep + 'and_smsmms' database = SQLiteDatabase(filename, temporary_directory=temporary_directory_path) try: database.Open(file_object) except (IOError, ValueError, sqlite3.DatabaseError) as exception: logger.warning( 'unable to open SQLite database with error: {0!s}'.format( exception)) file_object.close() return database_wal, wal_file_entry = self._OpenDatabaseWithWAL()
def Close(self): """Closes the database connection and cleans up the temporary file.""" self.schema = {} if self._is_open: self._database.close() self._database = None if os.path.exists(self._temp_db_file_path): try: os.remove(self._temp_db_file_path) except (OSError, IOError) as exception: logger.warning(( 'Unable to remove temporary copy: {0:s} of SQLite database: ' '{1:s} with error: {2!s}').format(self._temp_db_file_path, self._filename, exception)) self._temp_db_file_path = '' if os.path.exists(self._temp_wal_file_path): try: os.remove(self._temp_wal_file_path) except (OSError, IOError) as exception: logger.warning(( 'Unable to remove temporary copy: {0:s} of SQLite database: ' '{1:s} with error: {2!s}').format(self._temp_wal_file_path, self._filename, exception)) self._temp_wal_file_path = '' self._is_open = False
def get_last_pasties(self): # reset the pasties list pasties = [] # populate queue with data htmlPage, headers = download_url(self.archive_url) if not htmlPage: logger.warning("No HTML content for page {url}".format(url=self.archive_url)) return False pasties_ids = re.findall(self.archive_regex, htmlPage) if pasties_ids: for pastie_id in pasties_ids: # check if the pastie was already downloaded # and remember that we've seen it if self.seen_pastie(pastie_id): # do not append the seen things again in the queue continue # pastie was not downloaded yet. Add it to the queue if self.pastie_classname: class_name = globals()[self.pastie_classname] pastie = class_name(self, pastie_id) else: pastie = Pastie(self, pastie_id) pasties.append(pastie) return pasties logger.error("No last pasties matches for regular expression site:{site} regex:{regex}. Error in your regex? Dumping htmlPage \n {html}".format(site=self.name, regex=self.archive_regex, html=htmlPage.encode('utf8'))) return False
def LoadTargetFileToMemory(self, source_path_spec, configuration, file_path=None, file_spec=None, data_stream_name=None): try: if not file_spec: find_spec = file_system_searcher.FindSpec( case_sensitive=False, location=file_path, location_separator=source_path_spec.location) else: find_spec = file_spec except ValueError as exception: logger.error( 'Unable to build find specification for path: "{0:s}" with ' 'error: {1!s}'.format(file_path, exception)) path_spec_generator = self._path_spec_extractor.ExtractPathSpecs( [source_path_spec], find_specs=[find_spec], recurse_file_system=False, resolver_context=configuration.resolver_context) for path_spec in path_spec_generator: display_name = path_helper.PathHelper.GetDisplayNameForPathSpec( path_spec) try: file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=configuration.resolver_context) if file_entry is None or not file_entry.IsFile(): logger.warning( 'Unable to open file entry with path spec: {0:s}'. format(display_name)) return False if data_stream_name: file_object = file_entry.GetFileObject( data_stream_name=data_stream_name) if not file_object: return False return file_object elif not data_stream_name: file_object = file_entry.GetFileObject() if not file_object: return False return file_object except KeyboardInterrupt: return False
def BuildFindSpecs(self, paths, path_separator, environment_variables=None): """Builds find specifications from path filters. Args: path_filters (list[PathFilter]): path filters. environment_variables (Optional[list[EnvironmentVariableArtifact]]): environment variables. Returns: list[dfvfs.FindSpec]: find specifications for the file source type. """ find_specs = [] for path in paths: # Since paths are regular expression the path separator is escaped. if path_separator == '\\': path_separator = '\\\\' else: path_separator = path_separator expand_path = False path_segments = path.split(path_separator) for index, path_segment in enumerate(path_segments): if len(path_segment) <= 2: continue if path_segment[0] == '{' and path_segment[-1] == '}': # Rewrite legacy path expansion attributes, such as {systemroot} # into %SystemRoot%. path_segment = '%{0:s}%'.format(path_segment[1:-1]) path_segments[index] = path_segment if path_segment[0] == '%' and path_segment[-1] == '%': expand_path = True if expand_path: path_segments = path_helper.PathHelper.ExpandWindowsPathSegments( path_segments, environment_variables) if path_segments[0] != '': logger.warning( 'The path filter must be defined as an absolute path: ''{0:s}'.format(path)) continue # Strip the root path segment. path_segments.pop(0) if not path_segments[-1]: logger.warning( 'Empty last path segment in path: {0:s}'.format(path)) continue find_spec = file_system_searcher.FindSpec( case_sensitive=False, location_regex=path_segments) find_specs.append(find_spec) return find_specs
def RecursiveDirOrFileSearch(self, path_spec, output_path): display_name = path_helper.PathHelper.GetDisplayNameForPathSpec(path_spec) file_entry = path_spec_resolver.Resolver.OpenFileEntry(path_spec) if file_entry is None: logger.warning( 'Unable to open file entry with path spec: {0:s}'.format( display_name)) return if file_entry.IsDirectory(): if not os.path.exists(output_path + os.sep + file_entry.name): os.mkdir(output_path + os.sep + file_entry.name) for sub_file_entry in file_entry.sub_file_entries: try: if not sub_file_entry.IsAllocated(): continue except dfvfs_errors.BackEndError as exception: logger.warning( 'Unable to process file: {0:s} with error: {1!s}'.format( sub_file_entry.path_spec.comparable.replace( '\n', ';'), exception)) continue if sub_file_entry.type_indicator == dfvfs_definitions.TYPE_INDICATOR_TSK: if file_entry.IsRoot() and sub_file_entry.name == '$OrphanFiles': continue self.RecursiveDirOrFileSearch(sub_file_entry.path_spec, output_path + os.sep + file_entry.name) if file_entry.IsFile(): for data_stream in file_entry.data_streams: file_object = file_entry.GetFileObject(data_stream_name=data_stream.name) if not file_object: return False try: buffer_size = 65536 file = open(output_path + os.sep + file_entry.name, 'wb') file_object.seek(0, os.SEEK_SET) data = file_object.read(buffer_size) while data: file.write(data) data = file_object.read(buffer_size) file.close() except IOError as exception: print(display_name) logger.error( 'Failed to extract file "{0:s}" : {1!s}'.format(display_name, exception)) finally: file_object.close()
def scrol_right_to_left(self, retry=2, indx=None, ret_all_data=False): """ Scroll from right to left, @args: retry : Number of scroll retries after reaching to left indx : If you pass a low value it will try to scroll more, which result the big scrolling. ret_all_data : True if you want to get all data that found during the scroll, else False """ ele = "//android.widget.HorizontalScrollView[1]/" + \ "android.widget.LinearLayout[1]//android.widget.TextView[1]" for echtry in range(4): try: ele_obj = self.get_element_by_xpath(ele) break except Exception as excp: warning(str(excp)) if self.is_item_visible("setting-popup-cancel-btn"): info("cancel button shown up, clicking on it") self.click_on("setting-popup-cancel-btn") self.wait_for(1) else: raise Exception("not able to access the horizontal scroll element") list_text = list_text_old = [ech_ele.text for ech_ele in ele_obj] indx = len(ele_obj)/2 if indx == None else indx from_pt = ele_obj[0].location from_pt = (from_pt['x'], from_pt['y']) to_pt = ele_obj[indx].location to_pt = (to_pt['x'], to_pt['y']) tmp_retry = retry while tmp_retry > 0: self.scroll('H', frm_pt=from_pt, to_pt=to_pt) ele_obj = self.get_element_by_xpath(ele) list_text_new = [ech_ele.text for ech_ele in ele_obj] if list_text_new[:2] == list_text_old[:2]: tmp_retry -= 1 else: if ret_all_data: output = self.contains(list_text_old[:2], list_text_new) if output == False: info(str(list_text_old[:2]) + " not in " + \ str(list_text_new)) list_text[0:0] = list_text_new else: list_text[0:0] = list_text_new[:output[0]] list_text_old = list_text_new tmp_retry = retry if ret_all_data: return list_text
def clear_value(ele_obj): """ Clear out the element text @arg : ele_obj : element object that text need to clear out """ for i in range(7): try: txt = str(ele_obj.text).strip() if txt: ele_obj.click() ele_obj.clear() else: info("object got clear out") break except Exception as ex : warning("Got exception while clear out value :: " + str(ex))
def ExpandGlobStars(cls, path, path_separator): """Expands globstars "**" in a path. A globstar "**" will recursively match all files and zero or more directories and subdirectories. By default the maximum recursion depth is 10 subdirectories, a numeric values after the globstar, such as "**5", can be used to define the maximum recursion depth. Args: path (str): path to be expanded. path_separator (str): path segment separator. Returns: list[str]: String path expanded for each glob. """ expanded_paths = [] path_segments = path.split(path_separator) last_segment_index = len(path_segments) - 1 for segment_index, path_segment in enumerate(path_segments): recursion_depth = None if path_segment.startswith('**'): if len(path_segment) == 2: recursion_depth = 10 else: try: recursion_depth = int(path_segment[2:], 10) except (TypeError, ValueError): logger.warning(( 'Globstar with suffix "{0:s}" in path "{1:s}" not ' 'supported.').format(path_segment, path)) elif '**' in path_segment: logger.warning(( 'Globstar with prefix "{0:s}" in path "{1:s}" not ' 'supported.').format(path_segment, path)) if recursion_depth is not None: if recursion_depth <= 1 or recursion_depth > cls._RECURSIVE_GLOB_LIMIT: logger.warning(( 'Globstar "{0:s}" in path "{1:s}" exceed recursion maximum ' 'recursion depth, limiting to: {2:d}.').format( path_segment, path, cls._RECURSIVE_GLOB_LIMIT)) recursion_depth = cls._RECURSIVE_GLOB_LIMIT next_segment_index = segment_index + 1 for expanded_path_segment in [ ['*'] * depth for depth in range(1, recursion_depth + 1)]: expanded_path_segments = list(path_segments[:segment_index]) expanded_path_segments.extend(expanded_path_segment) if next_segment_index <= last_segment_index: expanded_path_segments.extend(path_segments[next_segment_index:]) expanded_path = path_separator.join(expanded_path_segments) expanded_paths.append(expanded_path) return expanded_paths or [path]
def ExtractTargetFileToPath(self, source_path_spec, configuration, file_path=None, file_spec=None, output_path=None, data_stream_name=None): # TODO: find_spec 있을 경우 처리 해야함. Load참조 try: if not file_spec: find_spec = file_system_searcher.FindSpec( case_sensitive=False, location=file_path, location_separator=source_path_spec.location) else: find_spec = file_spec except ValueError as exception: logger.error( 'Unable to build find specification for path: "{0:s}" with ' 'error: {1!s}'.format(file_path, exception)) path_spec_generator = self._path_spec_extractor.ExtractPathSpecs( [source_path_spec], find_specs=[find_spec], recurse_file_system=False, resolver_context=configuration.resolver_context) for path_spec in path_spec_generator: display_name = path_helper.PathHelper.GetDisplayNameForPathSpec( path_spec) try: file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=configuration.resolver_context) if file_entry is None or not file_entry.IsFile(): logger.warning( 'Unable to open file entry with path spec: {0:s}'. format(display_name)) return False if data_stream_name: file_object = file_entry.GetFileObject( data_stream_name=data_stream_name) if not file_object: return False try: buffer_size = 65536 file = open( output_path + os.sep + file_entry.name + '_' + data_stream_name, 'wb') file_object.seek(0, os.SEEK_SET) data = file_object.read(buffer_size) while data: file.write(data) data = file_object.read(buffer_size) file.close() except IOError as exception: # TODO: replace location by display name. location = getattr(file_entry.path_spec, 'location', '') logger.error( 'Failed to extract file "{0:s}" : {1!s}'.format( data_stream_name, exception)) return False finally: file_object.close() elif not data_stream_name: file_object = file_entry.GetFileObject() if not file_object: return False try: buffer_size = 65536 file = open(output_path + os.sep + file_entry.name, 'wb') file_object.seek(0, os.SEEK_SET) data = file_object.read(buffer_size) while data: file.write(data) data = file_object.read(buffer_size) file.close() except IOError as exception: logger.error( 'Failed to extract file "{0:s}" : {1!s}'.format( display_name, exception)) finally: file_object.close() except KeyboardInterrupt: return False
def message_handle(payload, emit): try: if 'id' not in payload: raise Exception("request id is not present") if 'command' not in payload: raise Exception("command is not present") command = payload['command'] if command == 'status': settings = read_config() logger.info("settings: [%s]", settings) found = glob.glob( f"{HOME}/.pm2/pids/hackrf-control-*") status = 'stopped' uptime = None if found: status = 'online' with open(found[0]) as fd: pid = fd.read() out = subprocess.check_output( f"stat /proc/{pid} | grep Modify", shell=True, encoding="utf-8") res = pattern.findall(out) uptime = res[0] if res else None emit( topic_res, { 'id': payload['id'], 'settings': settings, 'process': { 'status': status, 'uptime': uptime } }) elif command == 'logs': lines = payload.get('lines', 10) out = subprocess.check_output( f"tail {HOME}/.pm2/logs/hackrf-control-error.log -n {lines}", shell=True, encoding="utf-8") data = [] for x in out.split('\n'): created_at = x[0:23] pos = x.find(" ", 24) level = x[23:pos] pos = x.find(" ", pos + 1) content = x[pos:] data.append({ 'created_at': created_at, 'level': level, 'content': content }) emit(topic_res, { 'id': payload['id'], 'data': data }) elif command == 'config': if 'settings' not in payload: raise Exception("settings is not present") settings = payload['settings'] settings['_waveform'] = 'waveform' in settings logger.info("settings: [%s]", settings) save_config(settings) emit(topic_res, {'id': payload['id']}) else: emit(topic_res, {'id': payload['id']}) except Exception as ex: logger.warning("%s", payload) logger.error(ex) emit(topic_res, {'id': payload['id'], 'error': ex})
def download_url(url, data=None, cookie=None, loop_client=0, loop_server=0): # Client errors (40x): if more than 5 recursions, give up on URL (used for the 404 case) if loop_client >= retries_client: return None, None # Server errors (50x): if more than 100 recursions, give up on URL if loop_server >= retries_server: return None, None try: opener = None # Random Proxy if set in config random_proxy = get_random_proxy() if random_proxy: proxyh = urllib2.ProxyHandler({'http': random_proxy}) opener = urllib2.build_opener(proxyh, NoRedirectHandler()) # We need to create an opener if it didn't exist yet if not opener: opener = urllib2.build_opener(NoRedirectHandler()) # Random User-Agent if set in config user_agent = get_random_user_agent() opener.addheaders = [('Accept-Charset', 'utf-8')] if user_agent: opener.addheaders.append(('User-Agent', user_agent)) if cookie: opener.addheaders.append(('Cookie', cookie)) logger.debug( 'Downloading url: {url} with proxy: {proxy} and user-agent: {ua}'.format( url=url, proxy=random_proxy, ua=user_agent)) if data: response = opener.open(url, data) else: response = opener.open(url) htmlPage = unicode(response.read(), errors='replace') return htmlPage, response.headers except urllib2.HTTPError as e: failed_proxy(random_proxy) logger.warning("!!Proxy error on {0}.".format(url)) if 404 == e.code: htmlPage = e.read() logger.warning("404 from proxy received for {url}. Waiting 1 minute".format(url=url)) time.sleep(60) loop_client += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_client, total=retries_client, url=url)) return download_url(url, loop_client=loop_client) if 500 == e.code: htmlPage = e.read() logger.warning("500 from proxy received for {url}. Waiting 1 minute".format(url=url)) time.sleep(60) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server) if 504 == e.code: htmlPage = e.read() logger.warning("504 from proxy received for {url}. Waiting 1 minute".format(url=url)) time.sleep(60) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server) if 502 == e.code: htmlPage = e.read() logger.warning("502 from proxy received for {url}. Waiting 1 minute".format(url=url)) time.sleep(60) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server) if 403 == e.code: htmlPage = e.read() if 'Please slow down' in htmlPage or 'has temporarily blocked your computer' in htmlPage or 'blocked' in htmlPage: logger.warning("Slow down message received for {url}. Waiting 1 minute".format(url=url)) time.sleep(60) return download_url(url) logger.warning("ERROR: HTTP Error ##### {e} ######################## {url}".format(e=e, url=url)) return None, None except urllib2.URLError as e: logger.debug("ERROR: URL Error ##### {e} ######################## ".format(e=e, url=url)) if random_proxy: # remove proxy from the list if needed failed_proxy(random_proxy) logger.warning("Failed to download the page because of proxy error {0} trying again.".format(url)) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server) if 'timed out' in e.reason: logger.warning("Timed out or slow down for {url}. Waiting 1 minute".format(url=url)) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) time.sleep(60) return download_url(url, loop_server=loop_server) return None, None except socket.timeout: logger.debug("ERROR: timeout ############################# " + url) if random_proxy: # remove proxy from the list if needed failed_proxy(random_proxy) logger.warning("Failed to download the page because of socket error {0} trying again.".format(url)) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server) return None, None except Exception as e: failed_proxy(random_proxy) logger.warning("Failed to download the page because of other HTTPlib error proxy error {0} trying again.".format(url)) loop_server += 1 logger.warning("Retry {nb}/{total} for {url}".format(nb=loop_server, total=retries_server, url=url)) return download_url(url, loop_server=loop_server)
def message_handle(payload, emit): try: if 'id' not in payload: raise Exception("request id is not present") if 'command' not in payload: raise Exception("command is not present") command = payload['command'] if command == 'config': logger.debug( "Config Temperature %s", { 'id': payload['id'], 'status': 'Config max temperature', 'data': { 'temp': payload['temp'] } }) emit( topic_res, { 'id': payload['id'], 'status': 'Config max temperature', 'data': { 'temp': payload['temp'] } }) try: newTemp = str(payload['temp']) data = {"temp_max": newTemp} save_config(data) #updateTemperature('.local/config/hackrf-sensors.json', 0, '{"temp_max":' + newTemp + "}") except Exception as ex: logger.warning("%s", payload) logger.error(ex) emit(topic_res, { 'id': payload['id'], 'error': ex }) elif command == 'status': #INTEGRAR PROCESO QUE OBTIENE T° DE LA MAQUINA # Y PASARLO COMO VARIABLE A EMIT(TOPIC_RES) logger.debug("Getting sensors data") emit( topic_res, { 'id': payload['id'], 'status': 'Data sensors found', 'data': { 'temp': payload['temp'] } }) else: emit(topic_res, {'id': payload['id']}) except Exception as ex: logger.warning("%s", payload) logger.error(ex) emit(topic_res, {'id': payload['id'], 'error': ex})