def __init__(self, **kwargs): kwargs["src"] = "." # src-less module super().__init__(**kwargs) self.validate(kwargs) self.apiKey = kwargs.get("apiKey") self.type = kwargs.get("type") self.minify = str2bool(kwargs.get("minify")) # ip_lookup self.ip = kwargs.get("ip") self.port = int(kwargs.get("port", "0")) self.get_by_latest = str2bool(kwargs.get("getByLatest")) self.get_by_dataset_ts = kwargs.get("getByDatasetTs") self.get_missing_sets = str2bool( kwargs.get("getMissingDatasets")) # Inject additional paths self.get_all_missing_sets = str2bool( kwargs.get("getAllMissingDatasets")) # search_query self.query = kwargs.get("query") self.limit = int(kwargs.get("limit", "1")) self.retries = int(kwargs.get("retries", "5")) # KV-Store Init self.kv_store = kv_store(self.projectRoot, self.projectName) # Save current configs - Used for missingDatasets self.kwargs = kwargs
def __init__(self, **kwargs): super().__init__(**kwargs) self.validate(kwargs) # If subclass needs special variables define here self.type = kwargs.get("type") # General self.data_attribute = kwargs.get( "dataAttribute" ) # In which attribute to find the group value in the dataset self.group_name_attribute = kwargs.get( "groupNameAttribute" ) # In which attribute to find the group name in the dataset self.y_label = kwargs.get("ylabel", "") self.x_label = kwargs.get("xlabel", "") self.title = kwargs.get("title", "") self.line_style = kwargs.get("lineStyle", "-") self.marker_style = kwargs.get("markerStyle", "o") self.threshold = kwargs.get("threshold") self.figure_size = kwargs.get("figureSize", [20, 10]) self.filenames_regex_extract = kwargs.get("filenamesRegexExtract") self.color_map_ascending = kwargs.get("colormapAscending") self.color_map = kwargs.get("colormap") self.rotate_xticks = kwargs.get("rotateXTicks", 0) self.show_grid = str2bool(kwargs.get("showGrid")) self.show_legend = str2bool(kwargs.get("showLegend"))
def __init__(self, **kwargs): super().__init__(**kwargs) self.validate(kwargs) # all subclass special script attributes self.class_by = kwargs.get('classBy') self.classes = kwargs.get('classes') self.rules = kwargs.get('rules') self.keep_others = str2bool(kwargs.get('keepOthers')) self.multi_match = str2bool(kwargs.get('multiMatch'))
def __init__(self, **kwargs): super().__init__(**kwargs) self.validate(kwargs) # TBD REFACTOR MOVE UP THE SINGLE STRING TO LIST # If subclass needs special variables define here self.format = kwargs.get("format") self.keep = kwargs.get("keep") self.drop = kwargs.get("drop") self.ignoreMissingKeys = str2bool(kwargs.get("ignoreMissingKeys")) self.removeMissingKeys = str2bool(kwargs.get("removeMissingKeys"))
def test_utils_helpers_class(self): # Test File exists method self.assertEqual(fileExists("IAMNOTEXISTING.ccc"), False) self.assertEqual(fileExists(TESTDATA_OUT_FILENAME), True) # Test Str2Bool Method self.assertEqual(str2bool("yes"), True) self.assertEqual(str2bool("true"), True) self.assertEqual(str2bool("t"), True) self.assertEqual(str2bool("1"), True) self.assertEqual(str2bool("no"), False) self.assertEqual(str2bool("XYZ"), False) # Not a string given with self.assertRaises(ValueError): str2bool(100) # Test is_valid_url Method self.assertEqual(is_valid_url("THISISNOTSAURL.json"), False) self.assertEqual( is_valid_url("http://thisisavalidurlbutnotexisting1234.com"), True) self.assertEqual(is_valid_url("https://www.google.com"), True) # Test make_sha1 Method self.assertEqual(make_sha1("test"), "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3")
def __init__(self, **kwargs): super().__init__(**kwargs) self.validate(kwargs) self.alpha2_country_codes = iso3166.countries_by_alpha2 # If subclass needs special variables define here self.selected_map = kwargs.get("map") # Currently supported: 'europe', 'world' self.geojson_map = self.get_geojson_for_attribute(kwargs.get("map")) self.type = kwargs.get("type") # Type: Heatmap self.country_code_attribute = kwargs.get("countryCodeAttribute") self.grouped_value_attribute = kwargs.get("groupedValueAttribute") self.colormap = kwargs.get("colormap") self.display_legend = str2bool(kwargs.get("displayLegend")) self.display_labels = str2bool(kwargs.get("displayLabels")) self.labels_threshold = kwargs.get("labelsThreshold") self.title = kwargs.get("title")
def __init__(self, **kwargs): kwargs["src"] = "." # src-less module super().__init__(**kwargs) self.validate(kwargs) self.api_id = kwargs.get("apiId") self.api_secret = kwargs.get("apiSecret") self.series_id = kwargs.get("seriesId") self.file_id = kwargs.get("fileId") self.get_by_latest = str2bool(kwargs.get("getByLatest")) self.get_by_date = kwargs.get("getByDate") self.get_by_datasetId = kwargs.get("getByDatasetId") self.get_missing_sets = str2bool( kwargs.get("getMissingDatasets")) # Inject additional paths self.get_all_missing_sets = str2bool( kwargs.get("getAllMissingDatasets")) # KV-Store Init self.kv_store = kv_store(self.projectRoot, self.projectName) # Default Chunk Sizes if kwargs.get("chunkSizeDownload"): self.chunk_size_dl = kwargs.get("chunkSizeDownload") else: self.chunk_size_dl = DEFAULT_CHUNK_SIZE_DOWNLOAD if kwargs.get("chunkSizeDecomp"): self.chunk_size_dec = kwargs.get("chunkSizeDecomp") else: self.chunk_size_dec = DEFAULT_CHUNK_SIZE_DECOMPRESS # Save current configs - Used for missingDatasets self.kwargs = kwargs
def __init__(self, **kwargs): super().__init__(**kwargs) self.validate(kwargs) # If subclass needs special variables define here self.type = kwargs.get( "type" ) # Histogram, ComparedBarPlot, GroupedBarPlot, BarPlot3D, StackedBarPlot, GroupedBarPlot # General self.data_attribute = kwargs.get( "dataAttribute" ) # In which attribute to find the group value in the dataset self.group_name_attribute = kwargs.get( "groupNameAttribute" ) # In which attribute to find the group name in the dataset self.x_label = kwargs.get("xlabel", "") self.y_label = kwargs.get("ylabel", "") self.title = kwargs.get("title", "") self.threshold = kwargs.get("threshold") self.figure_size = kwargs.get("figureSize", [20, 10]) self.filenames_regex_extract = kwargs.get("filenamesRegexExtract") self.color_map_ascending = str2bool(kwargs.get("colormapAscending")) self.color_map = kwargs.get("colormap") self.rotate_xticks = kwargs.get("rotateXTicks", 0) self.rotate_yticks = kwargs.get("rotateYTicks", 0) self.horizontal = str2bool(kwargs.get("horizontal")) self.show_grid = str2bool(kwargs.get("showGrid")) self.show_legend = str2bool(kwargs.get("showLegend", True)) # Stacked Plots self.scaled_to_100 = str2bool(kwargs.get("scaledTo100")) # BarPlot3D self.z_label = kwargs.get("zlabel", "") self.show_y_axis_file_names = str2bool( kwargs.get("showYAxisFileNames"))
def validate(self, kwargs): """ Validates all arguments for the censys module. **Parameters**: kwargs : dict contains a dictionary of all attributes. """ super().validate(kwargs) self.cc_log("INFO", "Data Store Censys: started validation") # We want to get data via api, need to have api keys and wanted version if not kwargs.get("apiSecret"): raise ValidationError(self, ["apiSecret"], "Parameter cannot be empty!") if not kwargs.get("apiId"): raise ValidationError(self, ["apiId"], "Parameter cannot be empty") if not kwargs.get("seriesId"): raise ValidationError(self, ["seriesId"], "Parameter cannot be empty!") if not kwargs.get("fileId"): raise ValidationError( self, ["fileId"], "Parameter cannot be empty - define which file needs to be obtained!" ) if kwargs.get("getByLatest") and str2bool(kwargs.get("getByLatest")): pass elif kwargs.get("getByDate"): if not re.match('^\d{8}$', kwargs.get("getByDate")) or not isinstance( kwargs.get("getByDate"), str): raise ValidationError( self, ["getByDate"], "Check if getByDate is a string and correct format (valid ex.: '10042018')" ) try: self.parse_cc_currentdate_to_date(kwargs.get("getByDate")) except ValueError: raise ValidationError( self, ["getByDate"], "Was not able to parse getByDate has to e formated like '%d%m%Y'" ) elif kwargs.get("getByDatasetId"): if not re.match('^\d{8}T\d{4}$', kwargs.get("getByDatasetId")): raise ValidationError( self, ["getByDatasetId"], "Parameter getByDatasetId has to e formated like '20180410T1408'" ) else: raise ValidationError( self, ["getByLatest", "getByDate", "getByDatasetId"], "Please define one method how to get the data!") if kwargs.get("getMissingDatasets") and kwargs.get( "getAllMissingDatasets"): raise ValidationError( self, ["getMissingDatasets", "getAllMissingDatasets"], "Please only use one!") if kwargs.get("chunkSizeDownload"): try: kwargs["chunkSizeDownload"] = int( kwargs.get("chunkSizeDownload")) except ValueError: raise ValidationError( self, ["chunkSizeDownload"], "Chunk size download needs to be an integer!") if kwargs.get("chunkSizeDecomp"): try: kwargs["chunkSizeDecomp"] = int(kwargs.get("chunkSizeDecomp")) except ValueError: raise ValidationError( self, ["chunkSizeDecomp"], "Chunk size decompression needs to be an integer!") self.cc_log("INFO", "Data Store Censys: finished validation")