def create_dam_path(local_path, local_import_root, dam_import_root):
""" Returns <ok>, <path> """
ret = local_path.replace(local_import_root, dam_import_root)
log("=== 1 {}".format(ret))
ret = ret.replace("\\", "/")
log("=== 2 {}".format(ret))
return ret
def _post_file(server, filepath, dst_path, dry_run):
""" POST single file to DAM
curl -v -u admin:admin -X POST -i -F "file=@\"$FILENAME\"" $HOST$dampath.createasset.html &> $tempfile
"""
assert os.path.isfile(filepath)
if dry_run:
return OK
filename = os.path.basename(filepath)
f = open(filepath, 'rb')
mime, enc = mimetypes.guess_type(filepath)
log("Uploading {} as {}, {}".format(f, mime, enc))
form_data = dict(
file=(filename, f, mime, dict()),
fileName=filename
)
url = server.url("{path}.createasset.html".format(path=dst_path, filename=os.path.basename(filepath)))
log("POSTing to {}".format(url))
resp = requests.post(url, auth=server.auth, files=form_data)
if not _ok(resp.status_code):
error("Failed to upload file {}\n{}".format(filepath, resp.content))
return SERVER_ERROR
return OK
def clear_cache(server, options):
host_url = urlparse(server.host)
headers = {
'Host': host_url.hostname,
'CQ-Action': 'DELETE',
'CQ-Handle': '/',
'CQ-Path': '/'
}
host = server.dispatcher if server.dispatcher is not None else server.host
url = '{host}{path}'.format(host=host, path='/dispatcher/invalidate.cache')
log("Clearing cache with request to {}".format(url))
response = requests.get(url, auth=server.auth, headers=headers)
if response.status_code != 200:
sys.stderr.write("error: " + str(response.status_code) + "\n")
return SERVER_ERROR
if "<H1>OK</H1>" not in response.content:
sys.stderr.write("error: Failed to validate response '{}'\n".format(
response.content.strip()))
return SERVER_ERROR
if options.raw:
sys.stdout.write(response.content + "\n")
else:
sys.stdout.write("OK\n")
def clear_cache(server, options):
host_url = urlparse(server.host)
headers = {
'Host': host_url.hostname,
'CQ-Action': 'DELETE',
'CQ-Handle': '/',
'CQ-Path': '/'
}
host = server.dispatcher if server.dispatcher is not None else server.host
url = '{host}{path}'.format(host=host, path='/dispatcher/invalidate.cache')
log("Clearing cache with request to {}".format(url))
response = requests.get(url, auth=server.auth, headers=headers)
if response.status_code != 200:
sys.stderr.write("error: " + str(response.status_code) + "\n")
return SERVER_ERROR
if "<H1>OK</H1>" not in response.content:
sys.stderr.write("error: Failed to validate response '{}'\n".format(response.content.strip()))
return SERVER_ERROR
if options.raw:
sys.stdout.write(response.content + "\n")
else:
sys.stdout.write("OK\n")
def import_file(self, local_import_root, filepath):
""" Import single file """
assert os.path.isfile(filepath)
dam_path = get_dam_path(filepath, local_import_root,
self.destination_root)
log("Uplading {} to {}".format(filepath, dam_path))
if dam_path not in self.created_paths:
self.created_paths.add(dam_path)
if not self.dry_run:
self.api.create_folder(dam_path)
else:
log("Skipping creating dam path {}".format(dam_path))
status = OK
if not self.dry_run:
status = self.api.create_asset(filepath, dam_path)
if status == OK:
sys.stdout.write("{local} -> {dam}\n".format(local=filepath,
dam=dam_path))
sys.stdout.flush()
return status
def _list_assets(self, path):
""" List assets under path
:return (<status>, <list>)
<status>: <OK|USER_ERROR|SERVER_ERROR>
<list> [{name: <name>, title: <title>}...]
"""
log("Fetching asset list for {}".format(path))
req_path = "{root}{path}.json".format(root=API_ROOT, path=path)
url = self.server.url(req_path)
status, data = self._fetch_json(url)
if status != OK:
return SERVER_ERROR, None
_add_path(data, path)
next_url = _get_next_url(data)
while next_url is not None:
status, next_data = self._fetch_json(next_url)
if status != OK:
error("Failed to fetch next listing {}".format(next_url))
next_url = None
else:
_add_path(next_data, path)
data['entities'].extend(next_data['entities'])
next_url = _get_next_url(next_data)
return OK, data
def __init__(self, server, path, force_root=None):
if force_root is not None:
self.lock_dir = force_root
else:
self.lock_dir = os.path.join(ROOT_IMPORT_DIR,
self._hash_job(server, path))
log("Cache dir is {}".format(self.lock_dir))
def import_file(self, server, options, import_root, filepath):
""" Import single file """
assert os.path.isfile(filepath)
lock_file = self._lock_file(filepath)
if os.path.exists(lock_file):
sys.stdout.write("{i}/{n} Skipping {path}\n".format(i=self.current_file, n=self.total_files, path=filepath))
return OK
local_dir = os.path.dirname(filepath)
dest_dir = options.destination_root
if dest_dir is None:
dest_dir = os.path.join('/content/dam', os.path.basename(import_root))
dam_path = local_dir.replace(import_root, dest_dir)
log("Uplading {} to {}".format(filepath, dam_path))
if dam_path not in self.created_paths:
status = _create_dir(server, dam_path, options.dry_run)
if status != OK:
return status
self.created_paths.add(dam_path)
else:
log("Skipping creating dam path {}".format(dam_path))
status = _post_file(server, filepath, dam_path, options.dry_run)
if status == OK:
sys.stdout.write("{i}/{n} {local} -> {dam}\n".format(i=self.current_file, n=self.total_files,
local=filepath, dam=dam_path))
_touch(lock_file)
return status
def fetch_tree_synchronous(server, options, content_path):
""" Compount tool for fetching data immediately.
Performs: Create, Fetch, Wait, Remove
"""
task_id = create_task_id('rcp')
status, data = _create_task(server, task_id, content_path, content_path,
options)
if status != OK:
error("Faiiled to fetch {} from {}".format(content_path,
options.source_host))
return status
status, task = get_task_status(server, task_id)
log("Debug {}".format(json.dumps(task, indent=4)))
if task['status']['state'] != 'NEW':
error("Failed to locsate new task {}".format(task_id))
return SERVER_ERROR
status = start_task(server, task_id, options)
if status != OK:
error("Failed to fetch {} from {}, task {} did not start".format(
content_path, options.source_host, task_id))
return status
wait_for_task(server, task_id)
remove_task(server, task_id)
return OK
def stop_task(server, task_id, options):
""" Returns OK on success """
log("Stopping task {}".format(task_id))
status, content = post_command(server, task_id, 'stop')
if options.raw:
sys.stderr.write("{}\n".format(json.dumps(content)))
return status
def _create_task(server, task_id, src_path, dst_path, options):
""" Create a new task, does not start it """
log("Creating task {}".format(task_id))
payload = {
"cmd":
"create",
"id":
task_id,
"src":
"http://{credentials}@{source_host}/crx/-/jcr:root{content_path}".
format(credentials=options.source_credentials,
source_host=options.source_host,
content_path=src_path),
"dst":
dst_path,
"batchsize":
int(options.batch_size),
"update":
options.overwrite_existing,
"onlyNewer":
options.respect_modified_date,
"throttle":
int(options.throttle),
"recursive":
True,
}
url = server.url("/system/jackrabbit/filevault/rcp")
resp = requests.post(url, auth=server.auth, json=payload)
if resp.status_code != 201:
error("Failed to create task, request returned {} and {}".format(
resp.status_code, resp.content))
return SERVER_ERROR, None
return OK, resp.json()
def execute(self, server, argv):
log("Executing {}".format(self.name))
self.setopt(argv)
try:
ret = self.diff_subnodes(server, self.server2, self.path)
except KeyboardInterrupt:
return 0
def register_tool(_tool):
assert get_tool(_tool.name) is None
if _init_project is None:
name = _tool.name
else:
name = _init_project + ':' + _tool.name
log("Registering tool {}".format(name))
_tools[name] = _tool
def register_tool(_tool):
assert get_tool(_tool.name) is None
if _init_project is None:
name = _tool.name
else:
name = _init_project + ':' + _tool.name
log("Registering tool {}".format(name))
_tools[name] = _tool
def _touch(filename):
""" Create empty file """
par_dir = os.path.dirname(filename)
if not os.path.exists(par_dir):
log("Creating directory {}".format(par_dir))
os.makedirs(par_dir, mode=0755)
log("Creating lock file {}".format(filename))
open(filename, 'a').close()
def _fetch_json(self, url):
log("Fetching url {}".format(url))
resp = requests.get(url, auth=self.server.auth)
if resp.status_code != 200:
return SERVER_ERROR, None
data = json.loads(resp.content)
return OK, data
def _infer_type(val):
if val.lower() == 'true' or val.lower() == 'false':
log("Setting boolean typehint")
return 'Boolean'
elif val.isdigit():
log("Setting integer typehint")
return 'Long'
else:
return None
def start_bundle(server, bundlename, options):
""" curl -u admin:admin http://localhost:4505/system/console/bundles/org.apache.sling.scripting.jsp
-F action=start."""
form_data = {'action': 'start'}
url = server.url('/system/console/bundles/{bundle}'.format(bundle=bundlename))
log("POSTing to service {}".format(url))
resp = requests.post(url, auth=(server.username, server.password), data=form_data)
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
def deploy_bash_completion(paths=None):
""" Find the bash_completion.d directory and install the packaged script.
"""
path = _locate_bash_completion_dir(paths)
if path is not None:
acmd.log("Found bash completion script dir {}".format(path))
install_script(path)
return path
else:
sys.stderr.write("Could not find bash completion install dir.")
def get_bundle_list(server):
url = server.url('/system/console/bundles.json')
log("GETting service {}".format(url))
response = requests.get(url, auth=server.auth)
if response.status_code != 200:
error("Failed to list bundles: {}".format(response.status_code))
return []
bundles = response.json()['data']
return bundles
def _get_subnodes(server, path):
url = server.url("{path}.1.json".format(path=path))
log("GETting service {}".format(url))
resp = requests.get(url, auth=server.auth)
if resp.status_code != 200:
sys.stderr.write("error: Failed to get path {}, request returned {}\n".format(path, resp.status_code))
sys.exit(-1)
return resp.json()
def execute(self, server, argv):
options, args = parser.parse_args(argv)
if options.lock_dir is not None:
self.lock_dir = options.lock_dir
log("Cache dir is {}".format(self.lock_dir))
action = get_command(args)
actionarg = get_argument(args)
if action == 'import':
return self.import_path(server, options, actionarg)
return OK
def execute(self, server, argv):
log("Executing {}".format(self.name))
options, args = parser.parse_args(argv)
if len(args) >= 2:
path = args[1]
return list_node(server, options, path)
else:
ret = OK
for path in sys.stdin:
ret = ret | list_node(server, options, path.strip())
return ret
def _get_subnodes(server, path):
url = server.url("{path}.1.json".format(path=path))
log("GETting service {}".format(url))
resp = requests.get(url, auth=server.auth)
if resp.status_code != 200:
sys.stderr.write("error: Failed to get path {}, request returned {}\n".format(path, resp.status_code))
sys.exit(-1)
return resp.json()
def execute(self, server, argv):
log("Executing {}".format(self.name))
options, args = parser.parse_args(argv)
if len(args) >= 2:
path = args[1]
return list_node(server, options, path)
else:
ret = OK
for path in sys.stdin:
ret = ret | list_node(server, options, path.strip())
return ret
def _get_bash_version():
""" Find version of bash currently running. """
try:
full_text = subprocess.check_output(['bash', '--version'])
first_line = full_text.split('\n')[0]
version_string = first_line.lstrip("GNU bash, version ")
major = LooseVersion(version_string).version[0]
return major
except Exception as e:
acmd.log("No bash was found: " + e.message)
return 0
def delete_package(server, options, package_name):
url = get_package_url(package_name, server, options)
form_data = dict(cmd='delete')
log("Deleting package with POST to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if resp.status_code != 200:
error("Failed to delete package: {}".format(resp.content))
return SERVER_ERROR
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
return OK
def _get_task_list(server):
url = server.url("/system/jackrabbit/filevault/rcp")
log("GETting service {}".format(url))
resp = requests.get(url, auth=server.auth)
if resp.status_code != 200:
error("Failed to list rcp tasks, request returned {}".format(
resp.status_code))
return SERVER_ERROR, None
return OK, resp.json()
def parse_properties(props_str):
""" Translate command line properties string to dictionary.
example: prop0=value,prop1="Quoted value"
"""
log("Parsing props string '{}'".format(props_str))
ret = dict()
rest = props_str
while rest.strip() != '':
key, val, rest = _parse_property(rest)
log(" setting {}={}".format(key, val))
ret[key] = val
return ret
def delete_package(server, options, package_name):
url = get_package_url(package_name, server, options)
form_data = dict(cmd='delete')
log("Deleting package with POST to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if resp.status_code != 200:
error("Failed to delete package: {}".format(resp.content))
return SERVER_ERROR
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
return OK
def import_path(self, path):
""" Import generic file system path, could be file or dir """
if self.dry_run:
log("Dry running import")
if os.path.isdir(path):
return self.import_directory(path)
else:
import_root = os.path.dirname(path)
if self.destination_root is not None:
import_root = self.destination_root
return self.import_file(import_root, path)
def execute(self, server, argv):
self.api = AssetsApi(server)
options, args = parser.parse_args(argv)
action = get_action(args)
actionarg = get_argument(args)
if action == 'import':
funnel = AssetsImportFunnel(server, dry_run=options.dry_run, destination_root=options.destination_root)
return funnel.import_path(actionarg)
elif action == 'touch':
api = WorkflowsApi(server)
if len(args) >= 3:
self.touch_asset(api, actionarg, options.model)
else:
for line in sys.stdin:
self.touch_asset(api, line.strip(), options.model)
return OK
elif action == 'list' or action == 'ls':
status, data = self.api._list_assets(actionarg)
if status != OK:
return status
for item in data['entities']:
sys.stdout.write("{}\n".format(item['properties']['name']))
return OK
elif action == 'find':
status, data = self.api.find(actionarg)
if status != OK:
return status
for item in data:
props = item['properties']
path = acmd.jcr.path.join(props['path'], props['name'])
sys.stdout.write("{}\n".format(path))
return OK
elif action == 'tag':
tag_str = get_argument(args)
status, tags = parse_tags(tag_str)
if status != OK:
return status
if len(args) <= 3:
log("Reading files from input")
for path in sys.stdin:
path = path.strip()
self.tag_asset(path, tags)
else:
path = get_argument(args, i=3)
self.tag_asset(path, tags)
return OK
else:
error("Unknown action {}".format(action))
return USER_ERROR
def run(options, config, args, cmdargs):
tool_name, args = args[1], []
server = config.get_server(options.server)
if server is None:
sys.stderr.write("error: server '{srv}' not found.\n".format(srv=options.server))
return acmd.USER_ERROR
acmd.log("Using server {}".format(server))
cmd = acmd.get_tool(tool_name)
if cmd is None:
sys.stderr.write("error: tool '{cmd}' not found.\n".format(cmd=tool_name))
return acmd.USER_ERROR
else:
return cmd.execute(server, cmdargs)
def load_projects(projects):
""" Load any user specified tools directories.
Expecting dict of {<prefix>: <path>} """
ret = {}
for name, path in projects.items():
acmd.log("Loading project {}".format(name))
path = os.path.expanduser(path)
sys.path.insert(1, path)
init_file = os.path.join(path, '__init__.py')
acmd.set_current_project(name)
acmd.import_tools(init_file)
ret[name] = path
return ret
def create_dir(server, path, dry_run):
""" Create file in the DAM
e.g. curl -s -u admin:admin -X POST -F "jcr:primaryType=sling:OrderedFolder" $HOST$dampath > /dev/null
"""
if dry_run:
log("SKipping creating folder, dry run")
return
form_data = {'jcr:primaryType': 'sling:OrderedFolder'}
url = server.url(path)
log("POSTing to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if not _ok(resp.status_code):
raise AssetException("Failed to create directory {}\n{}".format(url, resp.content))
def create_dir(server, path, dry_run):
""" Create file in the DAM
e.g. curl -s -u admin:admin -X POST -F "jcr:primaryType=sling:OrderedFolder" $HOST$dampath > /dev/null
"""
if dry_run:
log("SKipping creating folder, dry run")
return
form_data = {'jcr:primaryType': 'sling:OrderedFolder'}
url = server.url(path)
log("POSTing to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if not _ok(resp.status_code):
raise AssetException("Failed to create directory {}\n{}".format(
url, resp.content))
def _create_dir(server, path, dry_run):
""" Create file in the DAM
e.g. curl -s -u admin:admin -X POST -F "jcr:primaryType=sling:OrderedFolder" $HOST$dampath > /dev/null
"""
if dry_run:
return OK
form_data = {'jcr:primaryType': 'sling:OrderedFolder'}
url = server.url(path)
log("POSTing to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if not _ok(resp.status_code):
error("Failed to create directory {}\n{}".format(url, resp.content))
return SERVER_ERROR
return OK
def get_subnodes(self, server, path):
url = server.url("{}.1.json".format(path))
log("GETting service {}".format(url))
resp = requests.get(url, auth=server.auth)
if self.options.raw:
print "{} {} {}".format(resp.url, resp.status_code, resp.encoding)
print "{}".format(resp.text)
if resp.status_code == 200:
print "{}".format(resp.json())
if resp.status_code != 200:
raise Exception(
"error: Failed to get path {}{}, request returned {}\n".format(server, path, resp.status_code)
)
return resp.json()
def install_package(server, options, package_name):
""" curl -u admin:admin -X POST \
http://localhost:4505/crx/packmgr/service/.json/etc/packages/export/name of package?cmd=install """
url = get_package_url(package_name, server, options)
form_data = dict(cmd="install")
log("Installing package with POST to {}".format(url))
resp = requests.post(url, auth=server.auth, data=form_data)
if resp.status_code != 200:
error("Failed to install package: {}".format(resp.content))
return SERVER_ERROR
data = resp.json()
assert data["success"] is True
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
else:
sys.stdout.write("{}\n".format(data["msg"]))
return OK
def start_workflow(server, options, model, path):
task_id = create_task_id(model)
form_data = dict(
model='/etc/workflow/models/{}/jcr:content/model'.format(model),
payload=_asset_path(path),
payloadType='JCR_PATH',
workflowTitle=task_id,
startComment=''
)
log(_asset_path(path))
url = server.url(INSTANCES_PATH)
resp = requests.post(url, auth=server.auth, data=form_data)
if resp.status_code != 201:
error("Unexpected error code {code}: {content}".format(
code=resp.status_code, content=resp.content))
return SERVER_ERROR
output = resp.content if options.raw else task_id
sys.stdout.write("{}\n".format(output))
return OK
def post_file(server, filepath, dst_path, dry_run):
""" POST single file to DAM
curl -v -u admin:admin -X POST -i -F "file=@\"$FILENAME\"" $HOST$dampath.createasset.html &> $tempfile
"""
assert os.path.isfile(filepath)
if dry_run:
return OK
filename = os.path.basename(filepath)
f = open(filepath, 'rb')
mime, enc = mimetypes.guess_type(filepath)
log("Uploading {} as {}, {}".format(f, mime, enc))
form_data = dict(
file=(filename, f, mime, dict()),
fileName=clean_path(filename)
)
url = server.url("{path}.createasset.html".format(path=dst_path, filename=os.path.basename(filepath)))
log("POSTing to {}".format(url))
resp = requests.post(url, auth=server.auth, files=form_data)
if not _ok(resp.status_code):
raise AssetException("Failed to upload file {}\n{}".format(filepath, resp.content))
return OK
def upload_package(server, options, filename):
""" curl -u admin:admin -F file=@"name of zip file" -F name="name of package"
-F force=true -F install=false http://localhost:4505/crx/packmgr/service.jsp """
form_data = dict(
file=(filename, open(filename, "rb"), "application/zip", dict()),
name=filename.rstrip(".zip"),
force=json_bool(options.install),
install=json_bool(options.install),
)
log(form_data)
url = server.url(SERVICE_PATH)
resp = requests.post(url, auth=server.auth, files=form_data)
if resp.status_code != 200:
error("Failed to upload paackage: {}: {}".format(resp.status_code, resp.content))
return SERVER_ERROR
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
else:
tree = ElementTree.fromstring(resp.content)
pkg_elem = tree.find("response").find("data").find("package")
pkg = parse_package(pkg_elem)
sys.stdout.write("{}\n".format(format_package(pkg)))
return OK
def activate(server, options, path):
""" curl -u admin:admin -X POST -F path="/content/path/to/page" -F cmd="activate" http://localhost:4502/bin/replicate.json
"""
url = server.url(SERVICE_PATH)
form_data = dict(cmd='activate',
path=path,
ignoredeactivated='false',
onlymodified='false')
resp = requests.post(url, auth=server.auth, data=form_data)
if not resp.status_code == 200:
error("Failed to perform activation because {}: {}\n".format(resp.status_code, resp.content))
else:
if options.raw:
sys.stdout.write("{}\n".format(resp.content))
else:
tree = html.fromstring(resp.text)
paths = tree.xpath("//body/div/div[@class='path']/text()")
paths = [path.split(' ')[0] for path in paths]
paths = filter(lambda path: path.strip() != '', paths)
msg = '\n'.join(paths)
log("{}".format(msg))
sys.stdout.write("{}\n".format(msg))
def split_argv(argv):
""" Split argument list into system arguments before the tool
and tool arguments afterwards.
['foo', 'bar', 'inspect', 'bink', 'bonk']
=> (['foo', 'bar', 'inspect'], ['inspect', 'bink', 'bonk'])"""
acmd.log("Splitting {}".format(argv))
for i, arg in enumerate(argv):
acmd.log("Checking for {}".format(arg))
if acmd.tool_repo.has_tool(arg):
left = argv[0:i + 1]
right = argv[i:]
acmd.log("Splitting args in {} and {}".format(left, right))
return left, right
return argv, []
