def run(self):
start_time = time.time()
files_to_process = self.definition['to_extract_files'].keys()
for files in self.format_input(files_to_process):
self.process_file(files)
print("--- %s seconds ---" % (time.time() - start_time))
module_format(self.definition['name'], type=1)
def __init__(self, definition):
# Module Definition
self.definition = definition
# Tar Filename
assert 'filename' in self.definition, "Tar file path not found!"
# Pretty Print Module Name
module_format(self.definition['name'])
def __init__(self, definition):
# Module Definition
self.definition = definition
# Pretty Print Module Name
module_format(self.definition['name'])
# From Aggregated info file, get the total users recorded
self._calculate_percentage(self.definition['percentage'])
# Initiate write file stream buffer
self.write_fileObj = open(SAMPLE_USERS_FILE, 'w')
def run(self):
start_time = time.time()
# Load sample users
assert check_fobj_exists(SAMPLE_USERS_FILE), "Sample users File not found in " + SAMPLE_USERS_FILE
SAMPLE_USERS = read_file(SAMPLE_USERS_FILE, header=SAMPLE_USERS_ID_FIELD)
# Run query
self._sample_users_review(SAMPLE_USERS)
print("--- %s seconds ---" % (time.time() - start_time))
module_format(self.definition['name'], type=1)
def __init__(self, definition):
# Module Definition
self.definition = definition
# Set number of lines to process at once
self.chunk_size = DEFAULT_LINES_CHUNK_SIZE
if 'chunk_lines' in self.definition:
self.chunk_size = self.definition['chunk_lines']
# Pretty Print Module Name
module_format(self.definition['name'])
# Check existence
self._delete_file_if_exists()
def __init__(self, definition):
# Module Definition
self.definition = definition
# Set number of lines to process at once
self.chunk_size = DEFAULT_LINES_CHUNK_SIZE
if 'chunk_lines' in self.definition:
self.chunk_size = self.definition['chunk_lines']
# Check for time range filters
assert 'time_filter' in self.definition, "'time_filter' is not specified!"
# Pretty Print Module Name
module_format(self.definition['name'])
# If present, delete folder. If not, create empty folder
self._execute_fs_initializations()
def run(self):
start_time = time.time()
self._sample_users()
print("--- %s seconds ---" % (time.time() - start_time))
module_format(self.definition['name'], type=1)