Exemplo n.º 1
0
 def test_update_library(self):
     service = Service([
         Stanza('package://boot', {}),
     ])
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertEqual(len(installed_packages), 1)
     self.assertEqual(installed_packages[0], 'boot')
     self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertEqual(len(installed_packages), 1)
     self.assertEqual(installed_packages[0], 'boot')
     self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed)
     r_packages.add(service, 'forecast')
     self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_not_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertTrue('boot' in installed_packages)
     self.assertTrue('forecast' in installed_packages)
     self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed)
     self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_installed)
     r_packages.remove(service, 'boot')
     self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertFalse('boot' in installed_packages)
     self.assertTrue('timeDate' in installed_packages)
     self.assertTrue('forecast' in installed_packages)
     self.assertEqual(r_packages.get_package_state('boot'), r_packages.metadata_package_not_installed)
     self.assertEqual(r_packages.get_package_state('timeDate'), r_packages.metadata_package_installed)
     self.assertEqual(r_packages.get_package_state('forecast'), r_packages.metadata_package_installed)
Exemplo n.º 2
0
 def test_update_library(self):
     service = Service([
         Stanza('package://boot', {}),
     ])
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertEqual(len(installed_packages), 1)
     self.assertEqual(installed_packages[0], 'boot')
     self.assertEqual(r_packages.get_package_state('boot'),
                      r_packages.metadata_package_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertEqual(len(installed_packages), 1)
     self.assertEqual(installed_packages[0], 'boot')
     self.assertEqual(r_packages.get_package_state('boot'),
                      r_packages.metadata_package_installed)
     r_packages.add(service, 'forecast')
     self.assertEqual(r_packages.get_package_state('forecast'),
                      r_packages.metadata_package_not_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertTrue('boot' in installed_packages)
     self.assertTrue('forecast' in installed_packages)
     self.assertEqual(r_packages.get_package_state('boot'),
                      r_packages.metadata_package_installed)
     self.assertEqual(r_packages.get_package_state('forecast'),
                      r_packages.metadata_package_installed)
     r_packages.remove(service, 'boot')
     self.assertEqual(r_packages.get_package_state('boot'),
                      r_packages.metadata_package_installed)
     r_packages.update_library(service)
     installed_packages = os.listdir(r_packages.get_library_path())
     self.assertFalse('boot' in installed_packages)
     self.assertTrue('timeDate' in installed_packages)
     self.assertTrue('forecast' in installed_packages)
     self.assertEqual(r_packages.get_package_state('boot'),
                      r_packages.metadata_package_not_installed)
     self.assertEqual(r_packages.get_package_state('timeDate'),
                      r_packages.metadata_package_installed)
     self.assertEqual(r_packages.get_package_state('forecast'),
                      r_packages.metadata_package_installed)
Exemplo n.º 3
0
 def test_execute(self):
     service = Service([
         Stanza('script://test1', {}),
         Stanza('package://test2', {}),
         Stanza('script://test3', {}),
     ])
     with tempfile.NamedTemporaryFile(delete=False) as f:
         output_csv_filename = f.name
     try:
         script = 'output = data.frame(a=c(1,2,3))\n'
         script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '")\n'
         r_framework.exeute(
             service,
             script,
             r_packages.get_library_path(),
             r_scripts.get_custom_scripts_path(),
         )
         with open(output_csv_filename, 'r') as f:
             reader = csv.reader(f)
             rows = [row for row in reader]
     finally:
         os.remove(output_csv_filename)
     self.assertEqual(len(rows), 4)
Exemplo n.º 4
0
 def test_execute(self):
     service = Service([
         Stanza('script://test1', {}),
         Stanza('package://test2', {}),
         Stanza('script://test3', {}),
     ])
     with tempfile.NamedTemporaryFile(delete=False) as f:
         output_csv_filename = f.name
     try:
         script = 'output = data.frame(a=c(1,2,3))\n'
         script += 'write.csv(output, file = "' + output_csv_filename.replace(
             '\\', '\\\\') + '")\n'
         r_framework.exeute(
             service,
             script,
             r_packages.get_library_path(),
             r_scripts.get_custom_scripts_path(),
         )
         with open(output_csv_filename, 'r') as f:
             reader = csv.reader(f)
             rows = [row for row in reader]
     finally:
         os.remove(output_csv_filename)
     self.assertEqual(len(rows), 4)
Exemplo n.º 5
0
def r(service, events, command_argument, fieldnames=None):
    if not events:
        events = []

    # collect field names
    if fieldnames is None:
        fieldnames = set()
        for result in events:
            for key in list(result.keys()):
                if key not in fieldnames:
                    fieldnames.add(key)
        if len(fieldnames) == 0:
            fieldnames = None

    r_id = str(uuid.uuid1())

    log({
        'r_id': r_id,
        'action': 'command',
        'phase': 'pre',
        'r_script': command_argument,
        'input_nb_events': len(events),
        'input_fieldnames': ', '.join(fieldnames) if fieldnames else '',
    })

    # installing prerequirements
    r_scripts.create_files(service)
    r_packages.update_library(service)

    input_csv_filename = None
    output_csv_filename = None
    output_library_usage_csv_filename = None
    try:
        # create CSV input file
        if fieldnames:
            with tempfile.NamedTemporaryFile(delete=False) as f:
                input_csv_filename = f.name
                writer = csv.DictWriter(f, fieldnames=list(fieldnames))
                writer.writeheader()
                writer.writerows(events)
        # create CSV output files
        with tempfile.NamedTemporaryFile(delete=False) as f:
            output_csv_filename = f.name
        with tempfile.NamedTemporaryFile(delete=False) as f:
            output_library_usage_csv_filename = f.name

        # create script file
        script = 'library.usage = data.frame(name=c()) \n'
        script += 'original_library = library \n'
        script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, '
        script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n'
        script += '{\n'
        script += '  if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n'
        script += '  library.usage <<- rbind(library.usage, data.frame(name=pkg))\n'
        script += '  return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,'
        script += '    warn.conflicts, quietly , verbose))\n'
        script += '}\n'
        script += 'library = new_library\n'

        # read content of csv file into input variable
        if input_csv_filename:
            options = 'check.names=FALSE, stringsAsFactors=FALSE'
            if options:
                options = options.strip()
            if options and len(options) > 0:
                options_with_comma = ', ' + options
            else:
                options_with_comma = ''
            script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \
                      + options_with_comma + ')\n'

        command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument)
        if command_argument_regex:
            script_name = command_argument_regex.group(1)
            script += 'source(\'' + script_name + '\')\n'
        else:
            script_content = command_argument
            script += script_content + '\n'

        # write content of output variable to csv file
        options = ''
        if options:
            options = options.strip()
        if options and len(options) > 0:
            options_with_comma = ', ' + options
        else:
            options_with_comma = ''
        script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\
                  + options_with_comma + ')\n'

        script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace(
            '\\', '\\\\') + '")\n'

        r_framework.exeute(
            service,
            script,
            r_packages.get_library_path(),
            r_scripts.get_custom_scripts_path(),
        )

        # read library usage
        with open(output_library_usage_csv_filename, "r") as f:
            reader = csv.reader(f)
            rows = [row for row in reader]
            package_names = set()
            if len(rows) > 0:
                header_row = rows[0]
                for row in rows[1:]:
                    event = {}
                    for i, cell in enumerate(row):
                        event[header_row[i]] = cell
                    package_name = event['name']
                    if package_name not in package_names:
                        package_names.add(package_name)
                        log({
                            'r_id': r_id,
                            'action': 'package_usage',
                            'package_name': package_name,
                        })

        # read csv output
        with open(output_csv_filename, "r") as f:
            reader = csv.reader(f)
            rows = [row for row in reader]
            if len(rows) == 0:
                return []
            header_row = rows[0]
            output = []
            for row in rows[1:]:
                event = {}
                for i, cell in enumerate(row):
                    event[header_row[i]] = cell
                output.append(event)

        log({
            'r_id': r_id,
            'action': 'command',
            'phase': 'post',
            'output_nb_events': len(output),
            'output_fieldnames': ', '.join(header_row) if header_row else '',
        })

        return header_row, output

    except Exception as e:
        log({
            'r_id': r_id,
            'action': 'command',
            'phase': 'exception',
            'exception': str(e),
        })
        raise

    # delete temp files
    finally:
        if input_csv_filename:
            os.remove(input_csv_filename)
        if output_csv_filename:
            os.remove(output_csv_filename)
        if output_library_usage_csv_filename:
            os.remove(output_library_usage_csv_filename)
Exemplo n.º 6
0
Arquivo: r.py Projeto: Ismael-VC/r
def r(service, events, command_argument, fieldnames=None):
    if not events:
        events = []

    # collect field names
    if fieldnames is None:
        fieldnames = set()
        for result in events:
            for key in list(result.keys()):
                if key not in fieldnames:
                    fieldnames.add(key)
        if len(fieldnames) == 0:
            fieldnames = None

    r_id = str(uuid.uuid1())

    log({
        'r_id': r_id,
        'action': 'command',
        'phase': 'pre',
        'r_script': command_argument,
        'input_nb_events': len(events),
        'input_fieldnames': ', '.join(fieldnames) if fieldnames else '',
        })

    # installing prerequirements
    r_scripts.create_files(service)
    r_packages.update_library(service)

    input_csv_filename = None
    output_csv_filename = None
    output_library_usage_csv_filename = None
    try:
        # create CSV input file
        if fieldnames:
            with tempfile.NamedTemporaryFile(delete=False) as f:
                input_csv_filename = f.name
                writer = csv.DictWriter(f, fieldnames=list(fieldnames))
                writer.writeheader()
                writer.writerows(events)
        # create CSV output files
        with tempfile.NamedTemporaryFile(delete=False) as f:
            output_csv_filename = f.name
        with tempfile.NamedTemporaryFile(delete=False) as f:
            output_library_usage_csv_filename = f.name

        # create script file
        script = 'library.usage = data.frame(name=c()) \n'
        script += 'original_library = library \n'
        script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, '
        script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n'
        script += '{\n'
        script += '  if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n'
        script += '  library.usage <<- rbind(library.usage, data.frame(name=pkg))\n'
        script += '  return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,'
        script += '    warn.conflicts, quietly , verbose))\n'
        script += '}\n'
        script += 'library = new_library\n'

        # read content of csv file into input variable
        if input_csv_filename:
            options = 'check.names=FALSE, stringsAsFactors=FALSE'
            if options:
                options = options.strip()
            if options and len(options) > 0:
                options_with_comma = ', '+options
            else:
                options_with_comma = ''
            script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \
                      + options_with_comma + ')\n'

        command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument)
        if command_argument_regex:
            script_name = command_argument_regex.group(1)
            script += 'source(\'' + script_name + '\')\n'
        else:
            script_content = command_argument
            script += script_content + '\n'

        # write content of output variable to csv file
        options = ''
        if options:
            options = options.strip()
        if options and len(options) > 0:
            options_with_comma = ', '+options
        else:
            options_with_comma = ''
        script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\
                  + options_with_comma + ')\n'

        script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace('\\', '\\\\') + '")\n'

        r_framework.exeute(
            service,
            script,
            r_packages.get_library_path(),
            r_scripts.get_custom_scripts_path(),
        )

        # read library usage
        with open(output_library_usage_csv_filename, "r") as f:
            reader = csv.reader(f)
            rows = [row for row in reader]
            package_names = set()
            if len(rows) > 0:
                header_row = rows[0]
                for row in rows[1:]:
                    event = {}
                    for i, cell in enumerate(row):
                        event[header_row[i]] = cell
                    package_name = event['name']
                    if package_name not in package_names:
                        package_names.add(package_name)
                        log({
                            'r_id': r_id,
                            'action': 'package_usage',
                            'package_name': package_name,
                            })

        # read csv output
        with open(output_csv_filename, "r") as f:
            reader = csv.reader(f)
            rows = [row for row in reader]
            if len(rows) == 0:
                return []
            header_row = rows[0]
            output = []
            for row in rows[1:]:
                event = {}
                for i, cell in enumerate(row):
                    event[header_row[i]] = cell
                output.append(event)

        log({
            'r_id': r_id,
            'action': 'command',
            'phase': 'post',
            'output_nb_events': len(output),
            'output_fieldnames': ', '.join(header_row) if header_row else '',
            })

        return header_row, output

    except Exception as e:
        log({
            'r_id': r_id,
            'action': 'command',
            'phase': 'exception',
            'exception': str(e),
            })
        raise

    # delete temp files
    finally:
        if input_csv_filename:
            os.remove(input_csv_filename)
        if output_csv_filename:
            os.remove(output_csv_filename)
        if output_library_usage_csv_filename:
            os.remove(output_library_usage_csv_filename)