def test_update_library(self):
service = Service([
Stanza('script://test', {
'content': 'YWRkIDwtIGZ1bmN0aW9uKGEsYikgew0KICBjID0gYStiDQogIHJldHVybihjKQ0KfQ0KcmVzdWx0ID0gY'
'WRkKDQsMjAwKQ0Kb3V0cHV0ID0gZGF0YS5mcmFtZShSZXN1bHQ9YyhyZXN1bHQpKQ==',
}),
])
r_scripts.create_files(service)
installed_scripts = os.listdir(r_scripts.get_custom_scripts_path())
self.assertEqual(len(installed_scripts), 1)
self.assertEqual(installed_scripts[0], 'test.r')
mtime = os.path.getmtime(os.path.join(r_scripts.get_custom_scripts_path(), installed_scripts[0]))
r_scripts.create_files(service)
self.assertEqual(
mtime,
os.path.getmtime(os.path.join(r_scripts.get_custom_scripts_path(), installed_scripts[0]))
)
time.sleep(1.5)
r_scripts.add(service, 'test', """
add <- function(a,b) {
c = a+b
return(c)
}
result = add(4,10)
output = data.frame(Result=c(result))
""")
time.sleep(1.5)
r_scripts.create_files(service)
installed_scripts = os.listdir(r_scripts.get_custom_scripts_path())
self.assertEqual(len(installed_scripts), 1)
self.assertEqual(installed_scripts[0], 'test.r')
self.assertNotEqual(
mtime,
os.path.getmtime(os.path.join(r_scripts.get_custom_scripts_path(), installed_scripts[0]))
)
mtime = os.path.getmtime(os.path.join(r_scripts.get_custom_scripts_path(), installed_scripts[0]))
r_scripts.create_files(service)
self.assertEqual(
mtime,
os.path.getmtime(os.path.join(r_scripts.get_custom_scripts_path(), installed_scripts[0]))
)
def test_execute(self):
service = Service([
Stanza('script://test1', {}),
Stanza('package://test2', {}),
Stanza('script://test3', {}),
])
with tempfile.NamedTemporaryFile(delete=False) as f:
output_csv_filename = f.name
try:
script = 'output = data.frame(a=c(1,2,3))\n'
script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '")\n'
r_framework.exeute(
service,
script,
r_packages.get_library_path(),
r_scripts.get_custom_scripts_path(),
)
with open(output_csv_filename, 'r') as f:
reader = csv.reader(f)
rows = [row for row in reader]
finally:
os.remove(output_csv_filename)
self.assertEqual(len(rows), 4)
def test_execute(self):
service = Service([
Stanza('script://test1', {}),
Stanza('package://test2', {}),
Stanza('script://test3', {}),
])
with tempfile.NamedTemporaryFile(delete=False) as f:
output_csv_filename = f.name
try:
script = 'output = data.frame(a=c(1,2,3))\n'
script += 'write.csv(output, file = "' + output_csv_filename.replace(
'\\', '\\\\') + '")\n'
r_framework.exeute(
service,
script,
r_packages.get_library_path(),
r_scripts.get_custom_scripts_path(),
)
with open(output_csv_filename, 'r') as f:
reader = csv.reader(f)
rows = [row for row in reader]
finally:
os.remove(output_csv_filename)
self.assertEqual(len(rows), 4)
def r(service, events, command_argument, fieldnames=None):
if not events:
events = []
# collect field names
if fieldnames is None:
fieldnames = set()
for result in events:
for key in list(result.keys()):
if key not in fieldnames:
fieldnames.add(key)
if len(fieldnames) == 0:
fieldnames = None
r_id = str(uuid.uuid1())
log({
'r_id': r_id,
'action': 'command',
'phase': 'pre',
'r_script': command_argument,
'input_nb_events': len(events),
'input_fieldnames': ', '.join(fieldnames) if fieldnames else '',
})
# installing prerequirements
r_scripts.create_files(service)
r_packages.update_library(service)
input_csv_filename = None
output_csv_filename = None
output_library_usage_csv_filename = None
try:
# create CSV input file
if fieldnames:
with tempfile.NamedTemporaryFile(delete=False) as f:
input_csv_filename = f.name
writer = csv.DictWriter(f, fieldnames=list(fieldnames))
writer.writeheader()
writer.writerows(events)
# create CSV output files
with tempfile.NamedTemporaryFile(delete=False) as f:
output_csv_filename = f.name
with tempfile.NamedTemporaryFile(delete=False) as f:
output_library_usage_csv_filename = f.name
# create script file
script = 'library.usage = data.frame(name=c()) \n'
script += 'original_library = library \n'
script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, '
script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n'
script += '{\n'
script += ' if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n'
script += ' library.usage <<- rbind(library.usage, data.frame(name=pkg))\n'
script += ' return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,'
script += ' warn.conflicts, quietly , verbose))\n'
script += '}\n'
script += 'library = new_library\n'
# read content of csv file into input variable
if input_csv_filename:
options = 'check.names=FALSE, stringsAsFactors=FALSE'
if options:
options = options.strip()
if options and len(options) > 0:
options_with_comma = ', ' + options
else:
options_with_comma = ''
script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \
+ options_with_comma + ')\n'
command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument)
if command_argument_regex:
script_name = command_argument_regex.group(1)
script += 'source(\'' + script_name + '\')\n'
else:
script_content = command_argument
script += script_content + '\n'
# write content of output variable to csv file
options = ''
if options:
options = options.strip()
if options and len(options) > 0:
options_with_comma = ', ' + options
else:
options_with_comma = ''
script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\
+ options_with_comma + ')\n'
script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace(
'\\', '\\\\') + '")\n'
r_framework.exeute(
service,
script,
r_packages.get_library_path(),
r_scripts.get_custom_scripts_path(),
)
# read library usage
with open(output_library_usage_csv_filename, "r") as f:
reader = csv.reader(f)
rows = [row for row in reader]
package_names = set()
if len(rows) > 0:
header_row = rows[0]
for row in rows[1:]:
event = {}
for i, cell in enumerate(row):
event[header_row[i]] = cell
package_name = event['name']
if package_name not in package_names:
package_names.add(package_name)
log({
'r_id': r_id,
'action': 'package_usage',
'package_name': package_name,
})
# read csv output
with open(output_csv_filename, "r") as f:
reader = csv.reader(f)
rows = [row for row in reader]
if len(rows) == 0:
return []
header_row = rows[0]
output = []
for row in rows[1:]:
event = {}
for i, cell in enumerate(row):
event[header_row[i]] = cell
output.append(event)
log({
'r_id': r_id,
'action': 'command',
'phase': 'post',
'output_nb_events': len(output),
'output_fieldnames': ', '.join(header_row) if header_row else '',
})
return header_row, output
except Exception as e:
log({
'r_id': r_id,
'action': 'command',
'phase': 'exception',
'exception': str(e),
})
raise
# delete temp files
finally:
if input_csv_filename:
os.remove(input_csv_filename)
if output_csv_filename:
os.remove(output_csv_filename)
if output_library_usage_csv_filename:
os.remove(output_library_usage_csv_filename)
def r(service, events, command_argument, fieldnames=None):
if not events:
events = []
# collect field names
if fieldnames is None:
fieldnames = set()
for result in events:
for key in list(result.keys()):
if key not in fieldnames:
fieldnames.add(key)
if len(fieldnames) == 0:
fieldnames = None
r_id = str(uuid.uuid1())
log({
'r_id': r_id,
'action': 'command',
'phase': 'pre',
'r_script': command_argument,
'input_nb_events': len(events),
'input_fieldnames': ', '.join(fieldnames) if fieldnames else '',
})
# installing prerequirements
r_scripts.create_files(service)
r_packages.update_library(service)
input_csv_filename = None
output_csv_filename = None
output_library_usage_csv_filename = None
try:
# create CSV input file
if fieldnames:
with tempfile.NamedTemporaryFile(delete=False) as f:
input_csv_filename = f.name
writer = csv.DictWriter(f, fieldnames=list(fieldnames))
writer.writeheader()
writer.writerows(events)
# create CSV output files
with tempfile.NamedTemporaryFile(delete=False) as f:
output_csv_filename = f.name
with tempfile.NamedTemporaryFile(delete=False) as f:
output_library_usage_csv_filename = f.name
# create script file
script = 'library.usage = data.frame(name=c()) \n'
script += 'original_library = library \n'
script += 'new_library <- function(pkg, help, pos = 2, lib.loc = NULL, character.only = FALSE, '
script += ' logical.return = FALSE, warn.conflicts = TRUE, quietly = FALSE, verbose = getOption("verbose"))\n'
script += '{\n'
script += ' if( is.symbol(substitute(pkg)) ) pkg=deparse(substitute(pkg))\n'
script += ' library.usage <<- rbind(library.usage, data.frame(name=pkg))\n'
script += ' return (original_library(package=pkg, help, pos, lib.loc, character.only=TRUE, logical.return,'
script += ' warn.conflicts, quietly , verbose))\n'
script += '}\n'
script += 'library = new_library\n'
# read content of csv file into input variable
if input_csv_filename:
options = 'check.names=FALSE, stringsAsFactors=FALSE'
if options:
options = options.strip()
if options and len(options) > 0:
options_with_comma = ', '+options
else:
options_with_comma = ''
script += 'input <- read.csv( "' + input_csv_filename.replace('\\', '\\\\') + '" ' \
+ options_with_comma + ')\n'
command_argument_regex = re.match(r'^(\w+\.[rR])$', command_argument)
if command_argument_regex:
script_name = command_argument_regex.group(1)
script += 'source(\'' + script_name + '\')\n'
else:
script_content = command_argument
script += script_content + '\n'
# write content of output variable to csv file
options = ''
if options:
options = options.strip()
if options and len(options) > 0:
options_with_comma = ', '+options
else:
options_with_comma = ''
script += 'write.csv(output, file = "' + output_csv_filename.replace('\\', '\\\\') + '" '\
+ options_with_comma + ')\n'
script += 'write.csv(library.usage, file = "' + output_library_usage_csv_filename.replace('\\', '\\\\') + '")\n'
r_framework.exeute(
service,
script,
r_packages.get_library_path(),
r_scripts.get_custom_scripts_path(),
)
# read library usage
with open(output_library_usage_csv_filename, "r") as f:
reader = csv.reader(f)
rows = [row for row in reader]
package_names = set()
if len(rows) > 0:
header_row = rows[0]
for row in rows[1:]:
event = {}
for i, cell in enumerate(row):
event[header_row[i]] = cell
package_name = event['name']
if package_name not in package_names:
package_names.add(package_name)
log({
'r_id': r_id,
'action': 'package_usage',
'package_name': package_name,
})
# read csv output
with open(output_csv_filename, "r") as f:
reader = csv.reader(f)
rows = [row for row in reader]
if len(rows) == 0:
return []
header_row = rows[0]
output = []
for row in rows[1:]:
event = {}
for i, cell in enumerate(row):
event[header_row[i]] = cell
output.append(event)
log({
'r_id': r_id,
'action': 'command',
'phase': 'post',
'output_nb_events': len(output),
'output_fieldnames': ', '.join(header_row) if header_row else '',
})
return header_row, output
except Exception as e:
log({
'r_id': r_id,
'action': 'command',
'phase': 'exception',
'exception': str(e),
})
raise
# delete temp files
finally:
if input_csv_filename:
os.remove(input_csv_filename)
if output_csv_filename:
os.remove(output_csv_filename)
if output_library_usage_csv_filename:
os.remove(output_library_usage_csv_filename)