def test_update_library(self): service = Service([ Stanza('package://boot', {}), ]) r_packages.update_library(service) installed_packages = os.listdir(r_packages.get_library_path()) self.assertEqual(len(installed_packages), 1) self.assertEqual(installed_packages[0], 'boot') self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed) r_packages.update_library(service) installed_packages = os.listdir(r_packages.get_library_path()) self.assertEqual(len(installed_packages), 1) self.assertEqual(installed_packages[0], 'boot') self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed) r_packages.add(service, 'forecast') self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_not_installed) r_packages.update_library(service) installed_packages = os.listdir(r_packages.get_library_path()) self.assertTrue('boot' in installed_packages) self.assertTrue('forecast' in installed_packages) self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed) self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_installed) r_packages.remove(service, 'boot') self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed) r_packages.update_library(service) installed_packages = os.listdir(r_packages.get_library_path()) self.assertFalse('boot' in installed_packages) self.assertTrue('timeDate' in installed_packages) self.assertTrue('forecast' in installed_packages) self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_not_installed) self.assertEqual(r_packages.get_package_state('timeDate'), r_packages.metadata_package_installed) self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_installed)
def r(service, events, command_argument, fieldnames=None): if not events: events = [] # collect field names if fieldnames is None: fieldnames = set() for result in events: for key in list(result.keys()): if key not in fieldnames: fieldnames.add(key) if len(fieldnames) == 0: fieldnames = None r_id = str(uuid.uuid1()) log({ 'r_id': r_id, 'action': 'command', 'phase': 'pre', 'r_script': command_argument, 'input_nb_events': len(events), 'input_fieldnames': ', '.join(fieldnames) if fieldnames else '', }) # installing prerequirements r_scripts.create_files(service) r_packages.update_library(service) input_csv_filename = None output_csv_filename = None output_library_usage_csv_filename = None try: # create CSV input file if fieldnames: with tempfile.NamedTemporaryFile(delete=False) as f: input_csv_filename = f.name writer = csv.DictWriter(f, fieldnames=list(fieldnames)) writer.writeheader() writer.writerows(events) # create CSV output files with tempfile.NamedTemporaryFile(delete=False) as f: output_csv_filename = f.name with tempfile.NamedTemporaryFile(delete=False) as f: output_library_usage_csv_filename = f.name # create script file script = 'library.usage = data.frame(name=c()) \n' script += 'original_library = library \n' script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, ' script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n' script += '{\n' script += ' if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n' script += ' library.usage <<- rbind(library.usage, data.frame(name=pkg))\n' script += ' return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,' script += ' warn.conflicts, quietly , verbose))\n' script += '}\n' script += 'library = new_library\n' # read content of csv file into input variable if input_csv_filename: options = 'check.names=FALSE, stringsAsFactors=FALSE' if options: options = options.strip() if options and len(options) > 0: options_with_comma = ', ' + options else: options_with_comma = '' script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \ + options_with_comma + ')\n' command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument) if command_argument_regex: script_name = command_argument_regex.group(1) script += 'source(\'' + script_name + '\')\n' else: script_content = command_argument script += script_content + '\n' # write content of output variable to csv file options = '' if options: options = options.strip() if options and len(options) > 0: options_with_comma = ', ' + options else: options_with_comma = '' script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\ + options_with_comma + ')\n' script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace( '\\', '\\\\') + '")\n' r_framework.exeute( service, script, r_packages.get_library_path(), r_scripts.get_custom_scripts_path(), ) # read library usage with open(output_library_usage_csv_filename, "r") as f: reader = csv.reader(f) rows = [row for row in reader] package_names = set() if len(rows) > 0: header_row = rows[0] for row in rows[1:]: event = {} for i, cell in enumerate(row): event[header_row[i]] = cell package_name = event['name'] if package_name not in package_names: package_names.add(package_name) log({ 'r_id': r_id, 'action': 'package_usage', 'package_name': package_name, }) # read csv output with open(output_csv_filename, "r") as f: reader = csv.reader(f) rows = [row for row in reader] if len(rows) == 0: return [] header_row = rows[0] output = [] for row in rows[1:]: event = {} for i, cell in enumerate(row): event[header_row[i]] = cell output.append(event) log({ 'r_id': r_id, 'action': 'command', 'phase': 'post', 'output_nb_events': len(output), 'output_fieldnames': ', '.join(header_row) if header_row else '', }) return header_row, output except Exception as e: log({ 'r_id': r_id, 'action': 'command', 'phase': 'exception', 'exception': str(e), }) raise # delete temp files finally: if input_csv_filename: os.remove(input_csv_filename) if output_csv_filename: os.remove(output_csv_filename) if output_library_usage_csv_filename: os.remove(output_library_usage_csv_filename)
def r(service, events, command_argument, fieldnames=None): if not events: events = [] # collect field names if fieldnames is None: fieldnames = set() for result in events: for key in list(result.keys()): if key not in fieldnames: fieldnames.add(key) if len(fieldnames) == 0: fieldnames = None r_id = str(uuid.uuid1()) log({ 'r_id': r_id, 'action': 'command', 'phase': 'pre', 'r_script': command_argument, 'input_nb_events': len(events), 'input_fieldnames': ', '.join(fieldnames) if fieldnames else '', }) # installing prerequirements r_scripts.create_files(service) r_packages.update_library(service) input_csv_filename = None output_csv_filename = None output_library_usage_csv_filename = None try: # create CSV input file if fieldnames: with tempfile.NamedTemporaryFile(delete=False) as f: input_csv_filename = f.name writer = csv.DictWriter(f, fieldnames=list(fieldnames)) writer.writeheader() writer.writerows(events) # create CSV output files with tempfile.NamedTemporaryFile(delete=False) as f: output_csv_filename = f.name with tempfile.NamedTemporaryFile(delete=False) as f: output_library_usage_csv_filename = f.name # create script file script = 'library.usage = data.frame(name=c()) \n' script += 'original_library = library \n' script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, ' script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n' script += '{\n' script += ' if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n' script += ' library.usage <<- rbind(library.usage, data.frame(name=pkg))\n' script += ' return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,' script += ' warn.conflicts, quietly , verbose))\n' script += '}\n' script += 'library = new_library\n' # read content of csv file into input variable if input_csv_filename: options = 'check.names=FALSE, stringsAsFactors=FALSE' if options: options = options.strip() if options and len(options) > 0: options_with_comma = ', '+options else: options_with_comma = '' script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \ + options_with_comma + ')\n' command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument) if command_argument_regex: script_name = command_argument_regex.group(1) script += 'source(\'' + script_name + '\')\n' else: script_content = command_argument script += script_content + '\n' # write content of output variable to csv file options = '' if options: options = options.strip() if options and len(options) > 0: options_with_comma = ', '+options else: options_with_comma = '' script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\ + options_with_comma + ')\n' script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace('\\', '\\\\') + '")\n' r_framework.exeute( service, script, r_packages.get_library_path(), r_scripts.get_custom_scripts_path(), ) # read library usage with open(output_library_usage_csv_filename, "r") as f: reader = csv.reader(f) rows = [row for row in reader] package_names = set() if len(rows) > 0: header_row = rows[0] for row in rows[1:]: event = {} for i, cell in enumerate(row): event[header_row[i]] = cell package_name = event['name'] if package_name not in package_names: package_names.add(package_name) log({ 'r_id': r_id, 'action': 'package_usage', 'package_name': package_name, }) # read csv output with open(output_csv_filename, "r") as f: reader = csv.reader(f) rows = [row for row in reader] if len(rows) == 0: return [] header_row = rows[0] output = [] for row in rows[1:]: event = {} for i, cell in enumerate(row): event[header_row[i]] = cell output.append(event) log({ 'r_id': r_id, 'action': 'command', 'phase': 'post', 'output_nb_events': len(output), 'output_fieldnames': ', '.join(header_row) if header_row else '', }) return header_row, output except Exception as e: log({ 'r_id': r_id, 'action': 'command', 'phase': 'exception', 'exception': str(e), }) raise # delete temp files finally: if input_csv_filename: os.remove(input_csv_filename) if output_csv_filename: os.remove(output_csv_filename) if output_library_usage_csv_filename: os.remove(output_library_usage_csv_filename)