def __init__(self, **kwargs):
kwargs["src"] = "." # src-less module
super().__init__(**kwargs)
self.validate(kwargs)
self.apiKey = kwargs.get("apiKey")
self.type = kwargs.get("type")
self.minify = str2bool(kwargs.get("minify"))
# ip_lookup
self.ip = kwargs.get("ip")
self.port = int(kwargs.get("port", "0"))
self.get_by_latest = str2bool(kwargs.get("getByLatest"))
self.get_by_dataset_ts = kwargs.get("getByDatasetTs")
self.get_missing_sets = str2bool(
kwargs.get("getMissingDatasets")) # Inject additional paths
self.get_all_missing_sets = str2bool(
kwargs.get("getAllMissingDatasets"))
# search_query
self.query = kwargs.get("query")
self.limit = int(kwargs.get("limit", "1"))
self.retries = int(kwargs.get("retries", "5"))
# KV-Store Init
self.kv_store = kv_store(self.projectRoot, self.projectName)
# Save current configs - Used for missingDatasets
self.kwargs = kwargs
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validate(kwargs)
# If subclass needs special variables define here
self.type = kwargs.get("type")
# General
self.data_attribute = kwargs.get(
"dataAttribute"
) # In which attribute to find the group value in the dataset
self.group_name_attribute = kwargs.get(
"groupNameAttribute"
) # In which attribute to find the group name in the dataset
self.y_label = kwargs.get("ylabel", "")
self.x_label = kwargs.get("xlabel", "")
self.title = kwargs.get("title", "")
self.line_style = kwargs.get("lineStyle", "-")
self.marker_style = kwargs.get("markerStyle", "o")
self.threshold = kwargs.get("threshold")
self.figure_size = kwargs.get("figureSize", [20, 10])
self.filenames_regex_extract = kwargs.get("filenamesRegexExtract")
self.color_map_ascending = kwargs.get("colormapAscending")
self.color_map = kwargs.get("colormap")
self.rotate_xticks = kwargs.get("rotateXTicks", 0)
self.show_grid = str2bool(kwargs.get("showGrid"))
self.show_legend = str2bool(kwargs.get("showLegend"))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validate(kwargs)
# all subclass special script attributes
self.class_by = kwargs.get('classBy')
self.classes = kwargs.get('classes')
self.rules = kwargs.get('rules')
self.keep_others = str2bool(kwargs.get('keepOthers'))
self.multi_match = str2bool(kwargs.get('multiMatch'))
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validate(kwargs)
# TBD REFACTOR MOVE UP THE SINGLE STRING TO LIST
# If subclass needs special variables define here
self.format = kwargs.get("format")
self.keep = kwargs.get("keep")
self.drop = kwargs.get("drop")
self.ignoreMissingKeys = str2bool(kwargs.get("ignoreMissingKeys"))
self.removeMissingKeys = str2bool(kwargs.get("removeMissingKeys"))
def test_utils_helpers_class(self):
# Test File exists method
self.assertEqual(fileExists("IAMNOTEXISTING.ccc"), False)
self.assertEqual(fileExists(TESTDATA_OUT_FILENAME), True)
# Test Str2Bool Method
self.assertEqual(str2bool("yes"), True)
self.assertEqual(str2bool("true"), True)
self.assertEqual(str2bool("t"), True)
self.assertEqual(str2bool("1"), True)
self.assertEqual(str2bool("no"), False)
self.assertEqual(str2bool("XYZ"), False)
# Not a string given
with self.assertRaises(ValueError):
str2bool(100)
# Test is_valid_url Method
self.assertEqual(is_valid_url("THISISNOTSAURL.json"), False)
self.assertEqual(
is_valid_url("http://thisisavalidurlbutnotexisting1234.com"), True)
self.assertEqual(is_valid_url("https://www.google.com"), True)
# Test make_sha1 Method
self.assertEqual(make_sha1("test"),
"a94a8fe5ccb19ba61c4c0873d391e987982fbbd3")
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validate(kwargs)
self.alpha2_country_codes = iso3166.countries_by_alpha2
# If subclass needs special variables define here
self.selected_map = kwargs.get("map") # Currently supported: 'europe', 'world'
self.geojson_map = self.get_geojson_for_attribute(kwargs.get("map"))
self.type = kwargs.get("type")
# Type: Heatmap
self.country_code_attribute = kwargs.get("countryCodeAttribute")
self.grouped_value_attribute = kwargs.get("groupedValueAttribute")
self.colormap = kwargs.get("colormap")
self.display_legend = str2bool(kwargs.get("displayLegend"))
self.display_labels = str2bool(kwargs.get("displayLabels"))
self.labels_threshold = kwargs.get("labelsThreshold")
self.title = kwargs.get("title")
def __init__(self, **kwargs):
kwargs["src"] = "." # src-less module
super().__init__(**kwargs)
self.validate(kwargs)
self.api_id = kwargs.get("apiId")
self.api_secret = kwargs.get("apiSecret")
self.series_id = kwargs.get("seriesId")
self.file_id = kwargs.get("fileId")
self.get_by_latest = str2bool(kwargs.get("getByLatest"))
self.get_by_date = kwargs.get("getByDate")
self.get_by_datasetId = kwargs.get("getByDatasetId")
self.get_missing_sets = str2bool(
kwargs.get("getMissingDatasets")) # Inject additional paths
self.get_all_missing_sets = str2bool(
kwargs.get("getAllMissingDatasets"))
# KV-Store Init
self.kv_store = kv_store(self.projectRoot, self.projectName)
# Default Chunk Sizes
if kwargs.get("chunkSizeDownload"):
self.chunk_size_dl = kwargs.get("chunkSizeDownload")
else:
self.chunk_size_dl = DEFAULT_CHUNK_SIZE_DOWNLOAD
if kwargs.get("chunkSizeDecomp"):
self.chunk_size_dec = kwargs.get("chunkSizeDecomp")
else:
self.chunk_size_dec = DEFAULT_CHUNK_SIZE_DECOMPRESS
# Save current configs - Used for missingDatasets
self.kwargs = kwargs
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.validate(kwargs)
# If subclass needs special variables define here
self.type = kwargs.get(
"type"
) # Histogram, ComparedBarPlot, GroupedBarPlot, BarPlot3D, StackedBarPlot, GroupedBarPlot
# General
self.data_attribute = kwargs.get(
"dataAttribute"
) # In which attribute to find the group value in the dataset
self.group_name_attribute = kwargs.get(
"groupNameAttribute"
) # In which attribute to find the group name in the dataset
self.x_label = kwargs.get("xlabel", "")
self.y_label = kwargs.get("ylabel", "")
self.title = kwargs.get("title", "")
self.threshold = kwargs.get("threshold")
self.figure_size = kwargs.get("figureSize", [20, 10])
self.filenames_regex_extract = kwargs.get("filenamesRegexExtract")
self.color_map_ascending = str2bool(kwargs.get("colormapAscending"))
self.color_map = kwargs.get("colormap")
self.rotate_xticks = kwargs.get("rotateXTicks", 0)
self.rotate_yticks = kwargs.get("rotateYTicks", 0)
self.horizontal = str2bool(kwargs.get("horizontal"))
self.show_grid = str2bool(kwargs.get("showGrid"))
self.show_legend = str2bool(kwargs.get("showLegend", True))
# Stacked Plots
self.scaled_to_100 = str2bool(kwargs.get("scaledTo100"))
# BarPlot3D
self.z_label = kwargs.get("zlabel", "")
self.show_y_axis_file_names = str2bool(
kwargs.get("showYAxisFileNames"))
def validate(self, kwargs):
"""
Validates all arguments for the censys module.
**Parameters**:
kwargs : dict
contains a dictionary of all attributes.
"""
super().validate(kwargs)
self.cc_log("INFO", "Data Store Censys: started validation")
# We want to get data via api, need to have api keys and wanted version
if not kwargs.get("apiSecret"):
raise ValidationError(self, ["apiSecret"],
"Parameter cannot be empty!")
if not kwargs.get("apiId"):
raise ValidationError(self, ["apiId"], "Parameter cannot be empty")
if not kwargs.get("seriesId"):
raise ValidationError(self, ["seriesId"],
"Parameter cannot be empty!")
if not kwargs.get("fileId"):
raise ValidationError(
self, ["fileId"],
"Parameter cannot be empty - define which file needs to be obtained!"
)
if kwargs.get("getByLatest") and str2bool(kwargs.get("getByLatest")):
pass
elif kwargs.get("getByDate"):
if not re.match('^\d{8}$',
kwargs.get("getByDate")) or not isinstance(
kwargs.get("getByDate"), str):
raise ValidationError(
self, ["getByDate"],
"Check if getByDate is a string and correct format (valid ex.: '10042018')"
)
try:
self.parse_cc_currentdate_to_date(kwargs.get("getByDate"))
except ValueError:
raise ValidationError(
self, ["getByDate"],
"Was not able to parse getByDate has to e formated like '%d%m%Y'"
)
elif kwargs.get("getByDatasetId"):
if not re.match('^\d{8}T\d{4}$', kwargs.get("getByDatasetId")):
raise ValidationError(
self, ["getByDatasetId"],
"Parameter getByDatasetId has to e formated like '20180410T1408'"
)
else:
raise ValidationError(
self, ["getByLatest", "getByDate", "getByDatasetId"],
"Please define one method how to get the data!")
if kwargs.get("getMissingDatasets") and kwargs.get(
"getAllMissingDatasets"):
raise ValidationError(
self, ["getMissingDatasets", "getAllMissingDatasets"],
"Please only use one!")
if kwargs.get("chunkSizeDownload"):
try:
kwargs["chunkSizeDownload"] = int(
kwargs.get("chunkSizeDownload"))
except ValueError:
raise ValidationError(
self, ["chunkSizeDownload"],
"Chunk size download needs to be an integer!")
if kwargs.get("chunkSizeDecomp"):
try:
kwargs["chunkSizeDecomp"] = int(kwargs.get("chunkSizeDecomp"))
except ValueError:
raise ValidationError(
self, ["chunkSizeDecomp"],
"Chunk size decompression needs to be an integer!")
self.cc_log("INFO", "Data Store Censys: finished validation")