def deploy_regression_tests():
_needs_targetenv()
target_dir = env.demo_data_root+"/regression"
if not exists(target_dir):
sudo("mkdir -p %(target_dir)s" % venv())
sudo("chown %(user)s %(target_dir)s" % venv())
rsync_project(target_dir, "%s/testfeeds/regression/" % env.projectroot, exclude=".*")
def _copy_local_repo(dataset_key, dataset_value):
rdf_example_path = dataset_value[0]
atom_example_subpath_and_file = dataset_value[1]
local('cp -r %s/documentation/exempel/documents/publ/' % env.projectroot + rdf_example_path +
'/* %(demodata_dir)s/%(dataset_key)s' % venv())
local('cp -r %s/documentation/exempel/feeds/sources/' % env.projectroot + atom_example_subpath_and_file +
' %(demodata_dir)s/%(dataset_key)s/feed.atom' % venv())
def install_regression_data(restart_tomcat=True):
""" Cleans main from system """
if restart_tomcat:
tomcat_stop()
sudo("rm -rf %(rinfo_main_store)s/" % venv())
sudo("mkdir %(rinfo_main_store)s/" % venv())
with lcd("%(rinfo_main_store)s/" % venv()):
sudo("tar xzvf /home/%(user)s/regression_test_data/rinfo_store.tar.gz" % venv())
sudo("chown tomcat:root -R %(rinfo_main_store)s/" % venv())
if restart_tomcat:
tomcat_start()
def deploy_dataset_war(dataset, restart=True):
"""Deploy a demo webapp for the given uploaded dataset."""
_can_handle_dataset(dataset)
if not exists(env.dist_dir):
run("mkdir %(dist_dir)s" % env)
if restart:
_deploy_war("%(java_packages)s/demodata-supply/target/%(dataset)s-demodata-supply.war" % venv(),
"%(dataset)s-demodata-supply" % venv())
else:
_deploy_war_norestart("%(java_packages)s/demodata-supply/target/%(dataset)s-demodata-supply.war" % venv(),
"%(dataset)s-demodata-supply" % venv())
def test():
_needs_targetenv()
report = JUnitReport()
url = "http://"+env.roledefs['main'][0]
test_url(report, "Verify feed exists and contains 'tag:lagrummet.se,2009:rinfo'", "main.feed",
"%(url)s/feed/current" % venv(), "tag:lagrummet.se,2009:rinfo")
test_url(report, "Verify dataset exists and contains 'tag:lagrummet.se,2009:rinfo'", "main.dataset",
"%(url)s/sys/dataset/rdf" % venv(), "tag:lagrummet.se,2009:rinfo")
if not report.empty():
file_name = "%(projectroot)s/testreport/main_test_report.log" % venv()
report.create_report(file_name)
print "Created report '%s'" % file_name
def test():
"""Http request to test admin is up and running correctly"""
report = JUnitReport()
url = "http://"+env.roledefs['admin'][0]
test_url(report, "Verify dataset exists and contains 'tag:lagrummet.se,2009:rinfo'", "admin.dataset",
"%(url)s/sys/dataset/rdf.rdf" % venv(), "tag:lagrummet.se,2009:rinfo")
test_url(report, "Verify current.atom exists and contains 'RInfo Base Data'", "admin.current",
"%(url)s/feed/current.atom" % venv(), "RInfo Base Data")
test_url(report, "Verify files index appears in root url", "admin.index", url, "Index" % venv())
if not report.empty():
file_name = "%(projectroot)s/testreport/admin_test_report.log" % venv()
report.create_report(file_name)
print "Created report '%s'" % file_name
def download(dataset, force="1"):
"""Downloads a demo dataset from its source."""
_can_handle_dataset(dataset)
if not int(force) and p.isdir("%(demodata_dir)s/%(dataset)s-raw" % venv()):
return
_mkdir_keep_prev("%(demodata_dir)s/%(dataset)s-raw" % venv())
with cd("%(demodata_dir)s/%(dataset)s-raw" % venv()):
if dataset in lagen_nu_datasets:
_download_lagen_nu_data(dataset)
elif dataset in riksdagen_se_datasets:
_download_riksdagen_data(dataset)
elif dataset in exempel_datasets_keys:
_copy_local_repo(dataset) # todo Fix this error with too few parameters
def _patch_catalina_properties():
# This will patch catalina.properties so that it contains the system
# property that controls where sesame stores its data
sesame_data_dir_key = "info.aduna.platform.appdata.basedir"
catalina_properties_path = "%(tomcat)s/conf/catalina.properties" % env
with settings(warn_only=True):
if sudo("grep '%(sesame_data_dir_key)s' %(catalina_properties_path)s" % venv()):
print "'%(sesame_data_dir_key)s' already present in %(catalina_properties_path)s" % venv()
else:
print "'%(sesame_data_dir_key)s' NOT found in %(catalina_properties_path)s" % venv()
print "Patching %(catalina_properties_path)s" % venv()
sudo("echo '# The data dir for Sesame used by rinfo-service' >> %(catalina_properties_path)s" % venv())
sudo("echo '%(sesame_data_dir_key)s=%(rinfo_rdf_repo_dir)s' >> %(catalina_properties_path)s" % venv())
def package(deps="1", test="1"):
if int(deps):
local_lib_rinfo_pkg(test)
_needs_targetenv()
flags = "" if int(test) else "-Dmaven.test.skip=true"
local("cd %(java_packages)s/rinfo-checker/ && "
"mvn %(flags)s -P%(target)s clean package war:war" % venv(), capture=False)
def clean():
""" Cleans checker from system. Will assume tomcat is inactive """
tomcat_stop()
#sudo("rm -rf %(tomcat_webapps)s/rinfo-service" % venv())
#sudo("rm -rf %(tomcat_webapps)s/rinfo-service.war" % venv())
sudo("rm -rf %(tomcat)s/logs/rinfo-service*.*" % venv())
tomcat_start()
def ping_start_collect_feed(default_feed=None):
_needs_targetenv()
if default_feed:
if not verify_url_content(" --data 'feed=%(default_feed)s' %(collector_url)s" % vars(),
"Scheduled collect of",
alternate_string_exists_in_content="is already scheduled for collect"):
print "Failed to start collect of '%s'" % default_feed
return
main_host_and_port = env.roledefs['main'][0] if env.target!='dev_unix' else "%s:8180" % env.roledefs['main'][0]
collector_url = "http://%s/collector" % main_host_and_port
if env.target=='regression':
feed_url = "http://%s/feed/current.atom" % env.roledefs['demosource'][0]
if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(), "Scheduled collect of"):
raise Exception("Scheduled collect failed")
else:
filename = "%(resources)s/%(target)s/datasources.n3" % venv()
read_file = open(filename, 'r')
for line in read_file:
line = line.strip()
if line.startswith('iana:current'):
start_index = line.index('<') + 1
end_index = line.index('>')
feed_url = line[start_index:end_index].strip()
if feed_url=='http://rinfo.lagrummet.se/feed/current':
continue
if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(),
"Scheduled collect of",
alternate_string_exists_in_content="is already scheduled for collect"):
print "Failed to start collect of '%s'" % feed_url
def upload(dataset):
"""Upload the transformed demo data depot to the demo server."""
_can_handle_dataset(dataset)
_needs_targetenv()
if not exists(env.demo_data_root):
sudo("mkdir -p %(demo_data_root)s" % env)
sudo("chown %(user)s %(demo_data_root)s" % env)
rsync_project(env.demo_data_root, "%(demodata_dir)s/%(dataset)s" % venv(), exclude=".*", delete=True)
def destroy_main_data(start_top_tomcat=True):
if not role_is_active('main'):
return
if start_top_tomcat:
tomcat_stop()
sudo("rm -rf %(rinfo_main_store)s/*" % venv())
if start_top_tomcat:
tomcat_start()
def create_depot(dataset):
"""Transforms the downloaded demo data to a depot."""
_can_handle_dataset(dataset)
_mkdir_keep_prev("%(demodata_dir)s/%(dataset)s" % venv())
if dataset in lagen_nu_datasets:
_transform_lagen_nu_data(dataset)
elif dataset in riksdagen_se_datasets:
_transform_riksdagen_data(dataset)
def package(deps="1", test="1"):
"""Builds and packages the rinfo-service war, configured for the target env."""
if int(deps):
app.local_lib_rinfo_pkg(test)
_needs_targetenv()
flags = "" if int(test) else "-Dmaven.test.skip=true"
local("cd %(java_packages)s/rinfo-service/ && "
"mvn %(flags)s -P%(target)s clean package war:war" % venv(), capture=False)
def clean():
""" Cleans main from system """
tomcat_stop()
#sudo("rm -rf %(tomcat_webapps)s/rinfo-main" % venv())
#sudo("rm -rf %(tomcat_webapps)s/rinfo-main.war" % venv())
sudo("rm -rf %(tomcat)s/logs/rinfo-main*.*" % venv())
#sudo("rm -rf %(rinfo_main_store)s/" % venv())
tomcat_start()
def update_hosts(ip="127.0.0.1", role=None):
if role and not role_is_active(role):
return
add_to_hosts = "\n%(ip)s %(target)s.lagrummet.se\n" \
"%(ip)s admin.%(target)s.lagrummet.se\n" \
"%(ip)s rinfo.%(target)s.lagrummet.se\n" \
"%(ip)s service.%(target)s.lagrummet.se\n" \
"%(ip)s checker.%(target)s.lagrummet.se\n" % venv()
append("/etc/hosts", add_to_hosts, use_sudo=True)
def package(source=None):
"""
Package the admin feed files into a servable directory.
"""
source = source or env.target
tg_sources = p.join(env.projectroot, "resources", source, "datasources.n3")
sourceopt = "-s " + fullpath(tg_sources) if p.exists(tg_sources) else ""
outdiropt = "-o " + fullpath(get_build_dir())
local("cd %(toolsdir)s/rinfomain && groovy base_as_feed.groovy "
"-b %(baseresources)s %(sourceopt)s %(outdiropt)s" % venv())
def build_dataset_war(dataset):
"""Build a webapp capable of serving an uploaded demo data depot."""
local("cd %(java_packages)s/demodata-supply && "
"mvn -Ddataset=%(dataset)s -Ddemodata-root=%(demo_data_root)s clean package" % venv(), capture=False)
def clean():
tomcat_stop()
sudo("rm -rf %(tomcat_webapps)s/rinfo-checker" % venv())
sudo("rm -rf %(tomcat_webapps)s/rinfo-checker.war" % venv())
sudo("rm -rf %(tomcat)s/logs/rinfo-checker*.*" % venv())
tomcat_start()
def _download_riksdagen_data(dataset):
local("%(java_opts)s groovy %(demodata_tools)s/data_riksdagen_se/fetch_data_riksdagen_se.groovy "
" %(demodata_dir)s/%(dataset)s-raw %(dataset)s -f" % venv())
def _transform_lagen_nu_data(dataset):
local("%(java_opts)s groovy %(demodata_tools)s/lagen_nu/n3dump_to_depot.groovy "
" %(demodata_dir)s/%(dataset)s-raw/lagennu-%(dataset)s.nt %(demodata_dir)s/%(dataset)s" % venv())
def _download_lagen_nu_data(dataset):
local("curl https://lagen.nu/%(dataset)s/parsed/rdf.nt"
" -o %(demodata_dir)s/%(dataset)s-raw/lagennu-%(dataset)s.nt" % venv())
def _deploy_war_norestart(localwar, warname, headless=False):
_needs_targetenv()
rsync_project("%(dist_dir)s/%(warname)s.war" % venv(), localwar, '--progress')
run("chmod 644 %(dist_dir)s/%(warname)s.war" % venv())
run("cp %(dist_dir)s/%(warname)s.war %(tomcat_webapps)s/." % venv())
run("touch %(tomcat_webapps)s/%(warname)s.war" % venv())
def test():
_needs_targetenv()
url = "http://"+env.roledefs['service'][0]
with lcd(env.projectroot+"/packages/java/rinfo-service/src/regression"):
local("casperjs test . --xunit=%(projectroot)s/testreport/service_test_report.log --url=%(url)s"
" --target=%(target)s --output=%(projectroot)s/testreport/" % venv())
def _transform_riksdagen_data(dataset):
local("%(java_opts)s groovy %(demodata_tools)s/data_riksdagen_se/depot_from_data_riksdagen_se.groovy "
" %(demodata_dir)s/%(dataset)s-raw %(demodata_dir)s/%(dataset)s" % venv())
def _deploy_war(localwar, warname, headless=False):
_needs_targetenv()
rsync_project("%(dist_dir)s/%(warname)s.war" % venv(), localwar, '--progress')
with _managed_tomcat_restart(5, headless):
run("rm -rf %(tomcat_webapps)s/%(warname)s/" % venv())
run("unzip -q %(dist_dir)s/%(warname)s.war -d %(tomcat_webapps)s/%(warname)s" % venv())
def _repotool(cmd):
repotool()
run("cd %(dist_dir)s && "
"java -jar %(rinfo_repo_jar)s %(cmd)s "
"%(rinfo_service_props)s rinfo.service.repo" % venv())
def local_lib_rinfo_pkg(test="1"):
flags = "" if int(test) else "-Dmaven.test.skip=true"
local("cd %(java_packages)s/ && mvn %(flags)s clean install " % venv(), capture=False)
def deploy_sesame():
setup()
package_sesame()
_patch_catalina_properties()
for warname in ['openrdf-sesame', 'sesame-workbench']:
app._deploy_war("%(local_sesame_dir)s/%(warname)s.war" % venv(), warname)