def install_resource(file_contents, templateURL, executor_user, packaged):
if packaged:
if isinstance(file_contents, basestring):
file_contents = StringIO(file_contents)
wgt_file = WgtFile(file_contents)
elif isinstance(file_contents, WgtFile):
wgt_file = file_contents
file_contents = wgt_file.get_underlying_file()
else:
raise Exception
template_contents = wgt_file.get_template()
else:
template_contents = file_contents
template = TemplateParser(template_contents)
resources = CatalogueResource.objects.filter(vendor=template.get_resource_vendor(), short_name=template.get_resource_name(), version=template.get_resource_version())[:1]
# Create/recover catalogue resource
if len(resources) == 1:
resource = resources[0]
else:
if packaged:
resource = add_widget_from_wgt(file_contents, executor_user, wgt_file=wgt_file)
else:
resource = add_resource_from_template(templateURL, template_contents, executor_user)
return resource
def wrapper(self, *args, **kwargs):
owner_user = User.objects.get(username=owner)
if shared:
base = self.shared_test_data_dir
else:
base = self.test_data_dir
with open(os.path.join(base, file_name), 'rb') as f:
wgt = WgtFile(f)
template = TemplateParser(wgt.get_template())
resource_info = template.get_resource_processed_info(process_urls=False)
if resource_info["type"] != 'mashup':
raise Exception
for embedded_resource in resource_info['embedded']:
if embedded_resource['src'].startswith('https://'):
resource_file = download_http_content(embedded_resource['src'])
else:
resource_file = BytesIO(wgt.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
install_resource_to_user(owner_user, file_contents=extra_resource_contents)
buildWorkspaceFromTemplate(template, owner_user)
return test_func(self, *args, **kwargs)
def update_resource_catalogue_cache(orm=None):
from wirecloud.catalogue.utils import wgt_deployer
if orm is not None:
resources = orm.CatalogueResource.objects.all()
else:
from wirecloud.catalogue.models import CatalogueResource
resources = CatalogueResource.objects.all()
for resource in resources:
try:
if resource.fromWGT:
base_dir = wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = wgt_file.get_template()
wgt_file.close()
else:
template = download_http_content(resource.template_uri)
template_parser = TemplateParser(template)
resource.json_description = json.dumps(template_parser.get_resource_info())
resource.save()
except TemplateParseException as e:
from django.conf import settings
if getattr(settings, 'WIRECLOUD_REMOVE_UNSUPPORTED_RESOURCES_MIGRATION', False) is not True:
raise e
print(' Removing %s' % (resource.vendor + '/' + resource.short_name + '/' + resource.version))
resource.delete()
def process(self, request, to_ws_id):
data = parse_json_request(request)
mashup_id = data.get('mashup', '')
workspace_id = data.get('workspace', '')
if mashup_id == '' and workspace_id == '':
return build_error_response(request, 422, _('Missing workspace or mashup parameter'))
elif mashup_id != '' and workspace_id != '':
return build_error_response(request, 422, _('Workspace and mashup parameters cannot be used at the same time'))
to_ws = get_object_or_404(Workspace, id=to_ws_id)
if not request.user.is_superuser and to_ws.creator != request.user:
return build_error_response(request, 403, _('You are not allowed to update this workspace'))
if mashup_id != '':
values = mashup_id.split('/', 3)
if len(values) != 3:
return build_error_response(request, 422, _('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(vendor=mashup_vendor, short_name=mashup_name, version=mashup_version)
if not resource.is_available_for(request.user) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
return build_error_response(request, 422, _('Mashup not found: %(mashup_id)s') % {'mashup_id': mashup_id})
base_dir = catalogue.wgt_deployer.get_base_dir(mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
else:
from_ws = get_object_or_404(Workspace, id=workspace_id)
if not request.user.is_superuser and from_ws.creator != request.user:
return build_error_response(request, 403, _('You are not allowed to read from workspace %s') % workspace_id)
options = {
'vendor': 'api',
'name': 'merge_op',
'version': '1.0',
'title': '',
'description': 'Temporal mashup for merging operation',
'email': '*****@*****.**',
}
template = TemplateParser(build_json_template_from_workspace(options, from_ws, from_ws.creator))
try:
check_mashup_dependencies(template, request.user)
except MissingDependencies as e:
details = {
'missingDependencies': e.missing_dependencies,
}
return build_error_response(request, 422, e, details=details)
fillWorkspaceUsingTemplate(to_ws, template)
return HttpResponse(status=204)
def add_widget_from_wgt(file, user, wgt_file=None, template=None, deploy_only=False):
close_wgt = False
if wgt_file is None:
wgt_file = WgtFile(file)
close_wgt = True
if template is None:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
resource_id = (
template.get_resource_vendor(),
template.get_resource_name(),
template.get_resource_version(),
)
file_name = '_'.join(resource_id) + '.wgt'
local_dir = wgt_deployer.get_base_dir(*resource_id)
local_wgt = os.path.join(local_dir, file_name)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
overrides = extract_resource_media_from_package(template, wgt_file, local_dir)
if close_wgt:
wgt_file.close()
f = open(local_wgt, "wb")
file.seek(0)
f.write(file.read())
f.close()
if not deploy_only:
return add_resource_from_template(file_name, template, user, fromWGT=True, overrides=overrides)
def wrapper(self, *args, **kwargs):
if shared:
base = self.shared_test_data_dir
else:
base = self.test_data_dir
final_creator = User.objects.get(username=creator) if creator is not None else None
final_users = tuple(User.objects.get(username=user) for user in users)
final_groups = tuple(Group.objects.get(name=group) for group in groups)
for resource in resources:
wgt_file = open(os.path.join(base, resource), 'rb')
wgt = WgtFile(wgt_file)
fix_dev_version(wgt, final_creator)
if deploy_only:
catalogue.add_packaged_resource(wgt.get_underlying_file(), final_creator, wgt_file=wgt, deploy_only=True)
wgt_file.close()
continue
resource = install_resource(wgt, final_creator)
if public:
resource.public = True
resource.save()
resource.users.add(*final_users)
resource.groups.add(*final_groups)
wgt_file.close()
return test_func(self, *args, **kwargs)
def _parse_ac_request(request):
fileURL = None
file_contents = None
content_type = get_content_type(request)[0]
try:
data = json.loads(request.body)
except Exception as e:
msg = _("malformed json data: %s") % unicode(e)
return build_error_response(request, 400, msg)
if 'url' not in data:
return build_error_response(request, 400, _('Missing widget URL'))
fileURL = data.get('url')
id_4CaaSt = data.get('4CaaStID')
if id_4CaaSt is None:
return build_error_response(request, 400, _('Missing 4CaaStID'))
if not isinstance(id_4CaaSt, string_types) or id_4CaaSt.strip() == '':
return build_error_response(request, 400, _('Invalid 4CaaStID'))
try:
downloaded_file = download_http_content(fileURL)
except:
return build_error_response(request, 409, _('Mashable application component could not be downloaded'))
downloaded_file = StringIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
# Create a custom version of the resource
template = TemplateParser(file_contents.get_template())
template_info = template.get_resource_info()
template_info['name'] += '@' + id_4CaaSt
for pref_name, pref_value in six.iteritems(data.get('preferences', {})):
for widget_pref_index, widget_pref in enumerate(template_info['preferences']):
if widget_pref['name'] == pref_name:
template_info['preferences'][widget_pref_index]['readonly'] = True
template_info['preferences'][widget_pref_index]['value'] = pref_value
break
# Write a new Wgt file
new_file = StringIO()
zin = zipfile.ZipFile(downloaded_file, 'r')
zout = zipfile.ZipFile(new_file, 'w')
zout.writestr('config.xml', write_rdf_description(template_info))
for item in zin.infolist():
if item.filename == 'config.xml':
continue
zout.writestr(item, zin.read(item.filename))
zin.close()
zout.close()
file_contents = WgtFile(new_file)
return id_4CaaSt, file_contents, fileURL
def _handle(self, *args, **options):
if len(args) < 1:
raise CommandError(_('Wrong number of arguments'))
self.verbosity = int(options.get('verbosity', 1))
users = []
groups = []
redeploy = options['redeploy']
public = options['public']
users_string = options['users'].strip()
groups_string = options['groups'].strip()
if redeploy is False and public is False and users_string == '' and groups_string == '':
raise CommandError(_('You must use at least one of the following flags: --redeploy, --users, --groups or --public '))
if not options['redeploy']:
if users_string != '':
for username in users_string.split(','):
users.append(User.objects.get(username=username))
if groups_string != '':
for groupname in groups_string.split(','):
groups.append(Group.objects.get(name=groupname))
for file_name in args:
try:
f = open(file_name, 'rb')
wgt_file = WgtFile(f)
except:
self.log(_('Failed to read from %(file_name)s') % {'file_name': file_name}, level=1)
continue
try:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
if options['redeploy']:
add_packaged_resource(f, None, wgt_file=wgt_file, template=template, deploy_only=True)
else:
for user in users:
install_resource_to_user(user, file_contents=wgt_file)
for group in groups:
install_resource_to_group(group, file_contents=wgt_file)
if public:
install_resource_to_all_users(file_contents=wgt_file)
wgt_file.close()
f.close()
self.log(_('Successfully imported \"%(name)s\" from \"%(file_name)s\"') % {'name': template.get_resource_processed_info()['title'], 'file_name': file_name}, level=1)
except:
self.log(_('Failed to import the mashable application component from %(file_name)s') % {'file_name': file_name}, level=1)
def add_packaged_resource(file, user, wgt_file=None, template=None, deploy_only=False):
close_wgt = False
if wgt_file is None:
wgt_file = WgtFile(file)
close_wgt = True
if template is None:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
resource_info = template.get_resource_info()
resource_id = (
resource_info['vendor'],
resource_info['name'],
resource_info['version'],
)
file_name = '_'.join(resource_id) + '.wgt'
check_packaged_resource(wgt_file, resource_info)
local_dir = wgt_deployer.get_base_dir(*resource_id)
local_wgt = os.path.join(local_dir, file_name)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
overrides = extract_resource_media_from_package(template, wgt_file, local_dir)
if close_wgt:
wgt_file.close()
f = open(local_wgt, "wb")
file.seek(0)
f.write(file.read())
f.close()
if not deploy_only:
resource_info.update(overrides)
resource = CatalogueResource.objects.create(
short_name=resource_info['name'],
vendor=resource_info['vendor'],
version=resource_info['version'],
type=CatalogueResource.RESOURCE_TYPES.index(resource_info['type']),
creator=user,
template_uri=file_name,
creation_date=now(),
popularity='0.0',
json_description=json.dumps(resource_info)
)
return resource
def create_workspace(owner, f=None, mashup=None, new_name=None, new_title=None, preferences={}, searchable=True, public=False):
from wirecloud.platform.workspace.mashupTemplateParser import buildWorkspaceFromTemplate
if mashup is not None and f is not None:
raise Exception
if f is not None:
wgt = f if isinstance(f, WgtFile) else WgtFile(f)
template = TemplateParser(wgt.get_template())
resource_info = template.get_resource_processed_info(process_urls=False)
if resource_info["type"] != 'mashup':
raise Exception
for embedded_resource in resource_info['embedded']:
if embedded_resource['src'].startswith('https://'):
resource_file = download_http_content(embedded_resource['src'])
else:
resource_file = BytesIO(wgt.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
install_resource_to_user(owner, file_contents=extra_resource_contents)
else:
values = mashup.split('/', 3)
if len(values) != 3:
raise TypeError(_('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(vendor=mashup_vendor, short_name=mashup_name, version=mashup_version)
if not resource.is_available_for(owner) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
raise Exception(_('Mashup not found: %(mashup)s') % {'mashup': mashup})
base_dir = catalogue.wgt_deployer.get_base_dir(mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
workspace, _foo = buildWorkspaceFromTemplate(template, owner, new_name=new_name, new_title=new_title, searchable=searchable, public=public)
if len(preferences) > 0:
update_workspace_preferences(workspace, preferences, invalidate_cache=False)
return workspace
def wrapper(self, *args, **kwargs):
owner_user = User.objects.get(username=owner)
if shared:
base = self.shared_test_data_dir
else:
base = self.test_data_dir
wgt_file = open(os.path.join(base, file_name), "rb")
wgt = WgtFile(wgt_file)
template = wgt.get_template()
buildWorkspaceFromTemplate(template, owner_user)
wgt_file.close()
return test_func(self, *args, **kwargs)
def add_widget_from_wgt(file, user, wgt_file=None, template=None, deploy_only=False):
close_wgt = False
if wgt_file is None:
wgt_file = WgtFile(file)
close_wgt = True
if template is None:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
if template.get_resource_type() == 'widget':
resource_info = template.get_resource_info()
code_url = resource_info['code_url']
if not code_url.startswith(('http://', 'https://')):
code = wgt_file.read(code_url)
try:
unicode(code, resource_info['code_charset'])
except UnicodeDecodeError:
msg = _('%(file_name)s was not encoded using the specified charset (%(charset)s according to the widget descriptor file).')
raise InvalidContents(msg % {'file_name': code_url, 'charset': resource_info['code_charset']})
resource_id = (
template.get_resource_vendor(),
template.get_resource_name(),
template.get_resource_version(),
)
file_name = '_'.join(resource_id) + '.wgt'
local_dir = wgt_deployer.get_base_dir(*resource_id)
local_wgt = os.path.join(local_dir, file_name)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
overrides = extract_resource_media_from_package(template, wgt_file, local_dir)
if close_wgt:
wgt_file.close()
f = open(local_wgt, "wb")
file.seek(0)
f.write(file.read())
f.close()
if not deploy_only:
return add_resource_from_template(file_name, template, user, fromWGT=True, overrides=overrides)
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError(_('Wrong number of arguments'))
user = None
if not options['deploy_only']:
user = User.objects.get(pk=1)
for file_name in args:
try:
f = open(file_name, 'rb')
wgt_file = WgtFile(f)
except:
print _('Failed to read from %(file_name)s') % {'file_name': file_name}
continue
try:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
try:
add_widget_from_wgt(f, user, wgt_file=wgt_file, template=template, deploy_only=options['deploy_only'])
except IntegrityError:
if not options['reinstall']:
raise
else:
old_resource = CatalogueResource.objects.get(vendor=template.get_resource_vendor(),
short_name=template.get_resource_name(),
version=template.get_resource_version()
)
delete_resource(old_resource, user)
add_widget_from_wgt(f, user, wgt_file=wgt_file, template=template)
wgt_file.close()
f.close()
print _('Successfully imported %(name)s widget') % {'name': template.get_resource_name()}
except IntegrityError:
print _('Version %(version)s of the %(name)s widget (from %(vendor)s) already exists') % {
'name': template.get_resource_name(),
'version': template.get_resource_version(),
'vendor': template.get_resource_vendor(),
}
except:
print _('Failed to import widget from %(file_name)s') % {'file_name': file_name}
def update_resource_catalogue_cache(orm=None):
if orm is not None:
resources = orm.CatalogueResource.objects.all()
else:
resources = CatalogueResource.objects.all()
resources_to_remove = []
for resource in resources:
try:
if getattr(resource, "fromWGT", True):
base_dir = wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = wgt_file.get_template()
wgt_file.close()
else:
# fromWGT attribute support was removed from Wirecloud in version 0.7.0
template = download_http_content(resource.template_uri)
template_parser = TemplateParser(template)
resource.json_description = json.dumps(template_parser.get_resource_info())
resource.save()
except (IOError, TemplateParseException) as e:
if isinstance(e, IOError) and e.errno != errno.ENOENT:
raise e
resources_to_remove.append(resource)
if (
len(resources_to_remove) > 0
and getattr(settings, "WIRECLOUD_REMOVE_UNSUPPORTED_RESOURCES_MIGRATION", False) is False
):
raise Exception(
"There are some mashable application components that are not supported anymore (use WIRECLOUD_REMOVE_UNSUPPORTED_RESOURCES_MIGRATION for removing automatically them in the migration process"
)
for resource in resources_to_remove:
print(" Removing %s" % (resource.vendor + "/" + resource.short_name + "/" + resource.version))
resource.delete()
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError(_('Wrong number of arguments'))
if not options['deploy_only']:
user = User.objects.get(pk=1)
else:
user = FakeUser('admin')
for file_name in args:
try:
wgt_file = WgtFile(file_name)
except:
print _('Failed to parse %(file_name)s') % {'file_name': file_name}
continue
try:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
try:
create_widget_from_wgt(wgt_file, user, deploy_only=options['deploy_only'])
except IntegrityError:
if not options['reinstall']:
raise
else:
deleteWidget(user, template.get_resource_name(),
template.get_resource_vendor(),
template.get_resource_version()
)
create_widget_from_wgt(wgt_file, user)
wgt_file.close()
print _('Successfully imported %(name)s widget') % {'name': template.get_resource_name()}
except IntegrityError:
print _('Version %(version)s of the %(name)s widget (from %(vendor)s) already exists') % {
'name': template.get_resource_name(),
'version': template.get_resource_version(),
'vendor': template.get_resource_vendor(),
}
except:
print _('Failed to import widget from %(file_name)s') % {'file_name': file_name}
def install_resource(file_contents, executor_user):
if isinstance(file_contents, string_types):
file_contents = BytesIO(file_contents)
wgt_file = WgtFile(file_contents)
elif isinstance(file_contents, WgtFile):
wgt_file = file_contents
file_contents = wgt_file.get_underlying_file()
else:
raise Exception
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
resources = CatalogueResource.objects.filter(vendor=template.get_resource_vendor(), short_name=template.get_resource_name(), version=template.get_resource_version())[:1]
# Create/recover catalogue resource
if len(resources) == 1:
resource = resources[0]
else:
resource = add_packaged_resource(file_contents, executor_user, wgt_file=wgt_file)
return resource
def update_resource_catalogue_cache(orm=None):
from wirecloud.catalogue.utils import wgt_deployer
if orm is not None:
resources = orm.CatalogueResource.objects.all()
else:
from wirecloud.catalogue.models import CatalogueResource
resources = CatalogueResource.objects.all()
for resource in resources:
if resource.fromWGT:
base_dir = wgt_deployer.get_base_dir(resource.vendor, resource.short_name, resource.version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = wgt_file.get_template()
wgt_file.close()
else:
template = download_http_content(resource.template_uri)
template_parser = TemplateParser(template)
resource.json_description = json.dumps(template_parser.get_resource_info())
resource.save()
def check_invalid_embedded_resources(wgt_file, resource_info):
if resource_info['type'] != 'mashup':
return
files = wgt_file.namelist()
for embedded_resource in resource_info['embedded']:
if embedded_resource['src'] not in files:
raise InvalidContents('Missing embedded file: %s' %
embedded_resource['src'])
embedded_wgt = WgtFile(BytesIO(wgt_file.read(
embedded_resource['src'])))
try:
check_packaged_resource(embedded_wgt)
except Exception as e:
raise InvalidContents('Invalid embedded file: %s' %
embedded_resource['src'],
details=e)
def test_invalid_packaged_widget_deployment(self):
wgt_file = WgtFile(
os.path.join(os.path.dirname(__file__), 'test-data',
'invalid_widget.wgt'))
catalogue_deployment_path = catalogue.wgt_deployer.get_base_dir(
'Morfeo', 'Test', '0.1')
deployment_path = wirecloud.platform.widget.utils.wgt_deployer.get_base_dir(
'Morfeo', 'Test', '0.1')
self.assertRaises(TemplateParseException, install_resource, wgt_file,
self.user)
self.assertRaises(CatalogueResource.DoesNotExist,
CatalogueResource.objects.get,
vendor='Morfeo',
short_name='Test',
version='0.1')
self.assertFalse(os.path.exists(deployment_path))
self.assertFalse(os.path.exists(catalogue_deployment_path))
def populate_component(self, wirecloud_user, log, vendor, name, version,
wgt):
if not CatalogueResource.objects.filter(
vendor=vendor, short_name=name, version=version).exists():
log('Installing the %(name)s widget... ' % {"name": name},
1,
ending='')
added, component = install_component(WgtFile(wgt),
executor_user=wirecloud_user,
users=[wirecloud_user])
IWidget.objects.filter(
widget__resource__vendor=vendor,
widget__resource__short_name=name).exclude(
widget__resource__version=version).update(
widget=component.widget,
widget_uri=component.local_uri_part)
log('DONE', 1)
return True
return False
def test_basic_packaged_mashup_deployment(self):
wgt_file = WgtFile(
os.path.join(os.path.dirname(wirecloud.commons.__file__),
'test-data', 'Wirecloud_PackagedTestMashup_1.0.zip'))
deployment_path = catalogue.wgt_deployer.get_base_dir(
'Wirecloud', 'PackagedTestMashup', '1.0')
install_resource_to_user(self.user, file_contents=wgt_file)
resource = CatalogueResource.objects.get(
vendor='Wirecloud', short_name='PackagedTestMashup', version='1.0')
self.assertTrue(os.path.isdir(deployment_path))
resource.delete()
self.assertRaises(CatalogueResource.DoesNotExist,
CatalogueResource.objects.get,
vendor='Wirecloud',
short_name='PackagedTestMashup',
version='1.0')
self.assertFalse(os.path.exists(deployment_path))
def prepare_temporal_resource_directories(cls):
cls.tmp_dir = mkdtemp()
# catalogue deployer
cls.old_catalogue_deployer = catalogue.wgt_deployer
cls.catalogue_tmp_dir_backup = os.path.join(cls.tmp_dir,
'catalogue_backup')
cls.catalogue_tmp_dir = os.path.join(cls.tmp_dir, 'catalogue')
catalogue.wgt_deployer = WgtDeployer(cls.catalogue_tmp_dir)
# showcase deployer
cls.old_deployer = showcase.wgt_deployer
cls.localcatalogue_tmp_dir_backup = os.path.join(cls.tmp_dir,
'localcatalogue_backup')
cls.localcatalogue_tmp_dir = os.path.join(cls.tmp_dir, 'localcatalogue')
showcase.wgt_deployer = WgtDeployer(cls.localcatalogue_tmp_dir)
# deploy resource files
for resource_file in cls.base_resources:
resource_file = open(
os.path.join(cls.shared_test_data_dir, resource_file), 'rb')
resource_wgt = WgtFile(resource_file)
catalogue.add_packaged_resource(resource_file,
None,
wgt_file=resource_wgt,
deploy_only=True)
showcase.wgt_deployer.deploy(resource_wgt)
resource_file.close()
# And freeze the resource files backup directories
if os.path.exists(cls.localcatalogue_tmp_dir):
os.rename(cls.localcatalogue_tmp_dir,
cls.localcatalogue_tmp_dir_backup)
else:
os.mkdir(cls.localcatalogue_tmp_dir_backup)
if os.path.exists(cls.catalogue_tmp_dir):
os.rename(cls.catalogue_tmp_dir, cls.catalogue_tmp_dir_backup)
else:
os.mkdir(cls.catalogue_tmp_dir_backup)
def test_basic_packaged_operator_deployment(self):
wgt_file = WgtFile(
os.path.join(os.path.dirname(wirecloud.commons.__file__),
'test-data', 'Wirecloud_TestOperator_1.0.zip'))
deployment_path = catalogue.wgt_deployer.get_base_dir(
'Wirecloud', 'TestOperator', '1.0')
install_component(wgt_file, users=[self.user])
resource = CatalogueResource.objects.get(vendor='Wirecloud',
short_name='TestOperator',
version='1.0')
self.assertTrue(os.path.isdir(deployment_path))
resource.delete()
self.assertRaises(CatalogueResource.DoesNotExist,
CatalogueResource.objects.get,
vendor='Wirecloud',
short_name='TestOperator',
version='1.0')
self.assertFalse(os.path.exists(deployment_path))
def _handle(self, *args, **options):
updated = False
self.verbosity = int(options.get('verbosity', 1))
with atomic():
wirecloud_user, created = User.objects.get_or_create(
username='******', defaults={'password': '******'})
if created:
updated = True
self.log('Creating a wirecloud user... DONE', 1)
if not CatalogueResource.objects.filter(
vendor="WireCloud",
short_name="workspace-browser",
version="0.1.1").exists():
updated = True
self.log('Installing the workspace-browser widget... ',
1,
ending='')
install_resource_to_user(
wirecloud_user, file_contents=WgtFile(WORKSPACE_BROWSER))
self.log('DONE', 1)
if not Workspace.objects.filter(creator__username="******",
name="home").exists():
updated = True
self.log(
'Creating a initial version of the wirecloud/home workspace... ',
1,
ending='')
with open(INITIAL_HOME_DASHBOARD_FILE, 'rb') as f:
workspace = create_workspace(wirecloud_user, f)
workspace.public = True
workspace.save()
self.log('DONE', 1)
if not updated:
self.log('Already up-to-date.', 1)
def wrapper(self, *args, **kwargs):
if shared:
base = self.shared_test_data_dir
else:
base = self.test_data_dir
for resource in resources:
wgt_file = open(os.path.join(base, resource), 'rb')
wgt = WgtFile(wgt_file)
if deploy_only:
catalogue.add_packaged_resource(wgt_file,
None,
wgt_file=wgt,
deploy_only=True)
wgt_file.close()
continue
resource = install_resource(wgt, None)
if public:
resource.public = True
resource.save()
final_users = (User.objects.get(username=user)
for user in users)
resource.users.add(*final_users)
final_groups = (Group.objects.get(name=group)
for group in groups)
resource.groups.add(*final_groups)
wgt_file.close()
return test_func(self, *args, **kwargs)
msg = _("malformed json data: %s") % unicode(e)
return build_error_response(request, 400, msg)
if 'url' not in data:
return build_error_response(request, 400, _('Missing widget URL'))
fileURL = data.get('url')
id_4CaaSt = data.get('4CaaStID')
try:
downloaded_file = downloader.download_http_content(fileURL)
except:
return build_error_response(request, 409, _('Widget content could not be downloaded'))
downloaded_file = StringIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
# Create a custom version of the resource
template = TemplateParser(file_contents.get_template())
template_info = template.get_resource_info()
template_info['name'] += '@' + id_4CaaSt
for pref_name, pref_value in data.get('preferences', {}).iteritems():
for widget_pref_index, widget_pref in enumerate(template_info['preferences']):
if widget_pref['name'] == pref_name:
template_info['preferences'][widget_pref_index]['readonly'] = True
template_info['preferences'][widget_pref_index]['value'] = pref_value
break
# Write a new Wgt file
new_file = StringIO()
def add_packaged_resource(file, user, wgt_file=None, template=None, deploy_only=False):
close_wgt = False
if wgt_file is None:
wgt_file = WgtFile(file)
close_wgt = True
if template is None:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
resource_info = template.get_resource_info()
if resource_info["type"] == "widget":
code_url = resource_info["contents"]["src"]
if not code_url.startswith(("http://", "https://")):
try:
code = wgt_file.read(code_url)
except KeyError:
msg = _("Missing contents file: %(file_name)s.")
raise InvalidContents(msg % {"file_name": code_url})
try:
code.decode(resource_info["contents"]["charset"])
except UnicodeDecodeError:
msg = _(
"%(file_name)s was not encoded using the specified charset (%(charset)s according to the widget descriptor file)."
)
raise InvalidContents(msg % {"file_name": code_url, "charset": resource_info["contents"]["charset"]})
check_invalid_doc_content(wgt_file, resource_info, "longdescription")
check_invalid_doc_content(wgt_file, resource_info, "doc")
check_invalid_doc_content(wgt_file, resource_info, "changelog")
resource_id = (template.get_resource_vendor(), template.get_resource_name(), template.get_resource_version())
file_name = "_".join(resource_id) + ".wgt"
local_dir = wgt_deployer.get_base_dir(*resource_id)
local_wgt = os.path.join(local_dir, file_name)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
overrides = extract_resource_media_from_package(template, wgt_file, local_dir)
if close_wgt:
wgt_file.close()
f = open(local_wgt, "wb")
file.seek(0)
f.write(file.read())
f.close()
if not deploy_only:
resource_info.update(overrides)
resource = CatalogueResource.objects.create(
short_name=resource_info["name"],
vendor=resource_info["vendor"],
version=resource_info["version"],
type=CatalogueResource.RESOURCE_TYPES.index(resource_info["type"]),
creator=user,
template_uri=file_name,
creation_date=now(),
popularity="0.0",
json_description=json.dumps(resource_info),
)
return resource
else:
values = mashup_id.split('/', 3)
if len(values) != 3:
return build_error_response(request, 422, _('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(vendor=mashup_vendor, short_name=mashup_name, version=mashup_version)
if not resource.is_available_for(request.user) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
return build_error_response(request, 422, _('Mashup not found: %(mashup_id)s') % {'mashup_id': mashup_id})
if resource.fromWGT:
base_dir = catalogue.wgt_deployer.get_base_dir(mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
else:
template = downloader.download_http_content(resource.template_uri, user=request.user)
try:
template = TemplateParser(template)
except:
build_error_response(request, 424, _('Downloaded invalid resource description from: %(url)s') % {'url': resource.template_uri})
try:
check_mashup_dependencies(template, request.user)
except MissingDependencies, e:
details = {
'missingDependencies': e.missing_dependencies,
}
return build_error_response(request, 422, unicode(e), details=details)
def add_packaged_resource(file, user, wgt_file=None, template=None, deploy_only=False):
close_wgt = False
if wgt_file is None:
wgt_file = WgtFile(file)
close_wgt = True
if template is None:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
resource_info = template.get_resource_info()
if resource_info['type'] == 'widget':
code_url = resource_info['contents']['src']
if not code_url.startswith(('http://', 'https://')):
try:
code = wgt_file.read(code_url)
except KeyError:
msg = _('Missing contents file: %(file_name)s.')
raise InvalidContents(msg % {'file_name': code_url})
try:
unicode(code, resource_info['contents']['charset'])
except UnicodeDecodeError:
msg = _('%(file_name)s was not encoded using the specified charset (%(charset)s according to the widget descriptor file).')
raise InvalidContents(msg % {'file_name': code_url, 'charset': resource_info['contents']['charset']})
resource_id = (
template.get_resource_vendor(),
template.get_resource_name(),
template.get_resource_version(),
)
file_name = '_'.join(resource_id) + '.wgt'
local_dir = wgt_deployer.get_base_dir(*resource_id)
local_wgt = os.path.join(local_dir, file_name)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
overrides = extract_resource_media_from_package(template, wgt_file, local_dir)
if close_wgt:
wgt_file.close()
f = open(local_wgt, "wb")
file.seek(0)
f.write(file.read())
f.close()
if not deploy_only:
resource_info.update(overrides)
resource = CatalogueResource.objects.create(
short_name=resource_info['name'],
vendor=resource_info['vendor'],
version=resource_info['version'],
type=CatalogueResource.RESOURCE_TYPES.index(resource_info['type']),
creator=user,
template_uri=file_name,
creation_date=now(),
popularity='0.0',
json_description=json.dumps(resource_info)
)
return resource
def populate_component(self, wirecloud_user, log, vendor, name, version, wgt):
if not CatalogueResource.objects.filter(vendor=vendor, short_name=name, version=version).exists():
updated = True
log('Installing the %(name)s widget... ' % {"name": name}, 1, ending='')
added, component = install_resource_to_user(wirecloud_user, file_contents=WgtFile(wgt))
IWidget.objects.filter(widget__resource__vendor=vendor, widget__resource__short_name=name).exclude(widget__resource__version=version).update(widget=component.widget, widget_uri=component.local_uri_part)
log('DONE', 1)
def create(self, request):
status_code = 201
force_create = False
install_embedded_resources = False
templateURL = None
file_contents = None
content_type = get_content_type(request)[0]
if content_type == 'multipart/form-data':
force_create = request.POST.get('force_create',
'false').strip().lower() == 'true'
public = request.POST.get('public',
'false').strip().lower() == 'true'
install_embedded_resources = request.POST.get(
'install_embedded_resources',
'false').strip().lower() == 'true'
if 'file' not in request.FILES:
return build_error_response(
request, 400, _('Missing component file in the request'))
downloaded_file = request.FILES['file']
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400, _('The uploaded file is not a zip file'))
elif content_type == 'application/octet-stream':
downloaded_file = BytesIO(request.body)
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400, _('The uploaded file is not a zip file'))
force_create = request.GET.get('force_create',
'false').strip().lower() == 'true'
public = request.GET.get('public',
'false').strip().lower() == 'true'
install_embedded_resources = request.GET.get(
'install_embedded_resources',
'false').strip().lower() == 'true'
else: # if content_type == 'application/json'
market_endpoint = None
data = parse_json_request(request)
install_embedded_resources = normalize_boolean_param(
request, 'install_embedded_resources',
data.get('install_embedded_resources', False))
force_create = data.get('force_create', False)
public = request.GET.get('public',
'false').strip().lower() == 'true'
templateURL = data.get('url')
market_endpoint = data.get('market_endpoint', None)
headers = data.get('headers', {})
if market_endpoint is not None:
if 'name' not in market_endpoint:
msg = _('Missing market name')
return build_error_response(request, 400, msg)
market_id = market_endpoint['name']
market_managers = get_market_managers(request.user)
if market_id not in market_managers:
return build_error_response(
request, 409,
_('Unknown market: %s') % market_id)
market_manager = market_managers[market_id]
downloaded_file = market_manager.download_resource(
request.user, templateURL, market_endpoint)
else:
try:
context = parse_context_from_referer(request)
except:
context = {}
try:
context["headers"] = CaseInsensitiveDict(headers)
response = WIRECLOUD_PROXY.do_request(
request, templateURL, "GET", context)
if response.status_code >= 300 or response.status_code < 200:
raise Exception()
downloaded_file = b''.join(response)
except:
return build_error_response(
request, 409,
_('Content cannot be downloaded from the specified url'
))
try:
downloaded_file = BytesIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400,
_('The file downloaded from the marketplace is not a zip file'
))
if public and not request.user.is_superuser:
return build_error_response(
request, 403,
_('You are not allowed to make resources publicly available to all users'
))
try:
fix_dev_version(file_contents, request.user)
added, resource = install_resource_to_user(
request.user,
file_contents=file_contents,
templateURL=templateURL)
if not added and force_create:
return build_error_response(request, 409,
_('Resource already exists'))
elif not added:
status_code = 200
if public:
install_resource_to_all_users(executor_user=request.user,
file_contents=file_contents)
except zipfile.BadZipfile as e:
return build_error_response(
request,
400,
_('The uploaded file is not a valid zip file'),
details="{}".format(e))
except OSError as e:
if e.errno == errno.EACCES:
return build_error_response(
request, 500,
_('Error writing the resource into the filesystem. Please, contact the server administrator.'
))
else:
raise
except TemplateParseException as e:
msg = "Error parsing config.xml descriptor file: %s" % e
details = "%s" % e
return build_error_response(request, 400, msg, details=details)
except (InvalidContents, UnsupportedFeature) as e:
details = e.details if hasattr(e, 'details') else None
return build_error_response(request,
400,
e,
details=six.text_type(details))
if install_embedded_resources:
info = {
'resource_details':
resource.get_processed_info(
request, url_pattern_name="wirecloud.showcase_media"),
'extra_resources': []
}
if resource.resource_type() == 'mashup':
resource_info = resource.get_processed_info(process_urls=False)
for embedded_resource in resource_info['embedded']:
resource_file = BytesIO(
file_contents.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
if public:
extra_resource_added, extra_resource = install_resource_to_user(
request.user,
file_contents=extra_resource_contents,
raise_conflicts=False)
else:
extra_resource_added, extra_resource = install_resource_to_user(
request.user,
file_contents=extra_resource_contents,
raise_conflicts=False)
if extra_resource_added:
info['extra_resources'].append(
extra_resource.get_processed_info(
request,
url_pattern_name="wirecloud.showcase_media"))
return HttpResponse(json.dumps(info, sort_keys=True),
status=status_code,
content_type='application/json; charset=UTF-8')
else:
return HttpResponse(json.dumps(resource.get_processed_info(
request, url_pattern_name="wirecloud.showcase_media"),
sort_keys=True),
status=status_code,
content_type='application/json; charset=UTF-8')
def create(self, request):
data = parse_json_request(request)
workspace_name = data.get('name', '').strip()
workspace_id = data.get('workspace', '')
mashup_id = data.get('mashup', '')
initial_pref_values = data.get('preferences', {})
allow_renaming = normalize_boolean_param(request, 'allow_renaming', data.get('allow_renaming', False))
dry_run = normalize_boolean_param(request, 'dry_run', data.get('dry_run', False))
if mashup_id == '' and workspace_id == '' and workspace_name == '':
return build_error_response(request, 422, _('Missing name parameter'))
elif mashup_id != '' and workspace_id != '':
return build_error_response(request, 422, _('Workspace and mashup parameters cannot be used at the same time'))
if mashup_id == '' and workspace_id == '':
if not is_valid_name(workspace_name):
return build_error_response(request, 422, _('invalid workspace name'))
if dry_run:
return HttpResponse(status=204)
try:
workspace = createEmptyWorkspace(workspace_name, request.user, allow_renaming=allow_renaming)
except IntegrityError:
msg = _('A workspace with the given name already exists')
return build_error_response(request, 409, msg)
else:
if mashup_id != '':
values = mashup_id.split('/', 3)
if len(values) != 3:
return build_error_response(request, 422, _('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(vendor=mashup_vendor, short_name=mashup_name, version=mashup_version)
if not resource.is_available_for(request.user) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
return build_error_response(request, 422, _('Mashup not found: %(mashup_id)s') % {'mashup_id': mashup_id})
base_dir = catalogue.wgt_deployer.get_base_dir(mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
else:
from_ws = get_object_or_404(Workspace, id=workspace_id)
if from_ws.public is False and not request.user.is_superuser and from_ws.creator != request.user:
return build_error_response(request, 403, _('You are not allowed to read from workspace %s') % workspace_id)
options = {
'vendor': 'api',
'name': from_ws.name,
'version': '1.0',
'title': '',
'description': 'Temporal mashup for the workspace copy operation',
'email': '*****@*****.**',
}
template = TemplateParser(build_json_template_from_workspace(options, from_ws, from_ws.creator))
try:
check_mashup_dependencies(template, request.user)
except MissingDependencies as e:
details = {
'missingDependencies': e.missing_dependencies,
}
return build_error_response(request, 422, e, details=details)
if dry_run:
return HttpResponse(status=204)
if workspace_name == '':
workspace_name = None
try:
workspace, _junk = buildWorkspaceFromTemplate(template, request.user, allow_renaming=allow_renaming, new_name=workspace_name)
except IntegrityError:
msg = _('A workspace with the given name already exists')
return build_error_response(request, 409, msg)
if len(initial_pref_values) > 0:
update_workspace_preferences(workspace, initial_pref_values, invalidate_cache=False)
workspace_data = get_global_workspace_data(workspace, request.user)
return workspace_data.get_response(status_code=201, cacheable=False)
def create_workspace(owner,
f=None,
mashup=None,
new_name=None,
new_title=None,
preferences={},
searchable=True,
public=False):
from wirecloud.platform.workspace.mashupTemplateParser import buildWorkspaceFromTemplate
if mashup is not None and f is not None:
raise Exception
if f is not None:
wgt = f if isinstance(f, WgtFile) else WgtFile(f)
template = TemplateParser(wgt.get_template())
resource_info = template.get_resource_processed_info(
process_urls=False)
if resource_info["type"] != 'mashup':
raise Exception
for embedded_resource in resource_info['embedded']:
if embedded_resource['src'].startswith('https://'):
resource_file = download_http_content(embedded_resource['src'])
else:
resource_file = BytesIO(wgt.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
install_resource_to_user(owner,
file_contents=extra_resource_contents)
else:
values = mashup.split('/', 3)
if len(values) != 3:
raise TypeError(_('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(vendor=mashup_vendor,
short_name=mashup_name,
version=mashup_version)
if not resource.is_available_for(
owner) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
raise Exception(
_('Mashup not found: %(mashup)s') % {'mashup': mashup})
base_dir = catalogue.wgt_deployer.get_base_dir(mashup_vendor,
mashup_name,
mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
workspace, _foo = buildWorkspaceFromTemplate(template,
owner,
new_name=new_name,
new_title=new_title,
searchable=searchable,
public=public)
if len(preferences) > 0:
update_workspace_preferences(workspace,
preferences,
invalidate_cache=False)
return workspace
def create(self, request):
force_create = False
install_embedded_resources = False
templateURL = None
file_contents = None
content_type = get_content_type(request)[0]
if content_type == "multipart/form-data":
force_create = request.POST.get("force_create", "false").strip().lower() == "true"
install_embedded_resources = (
request.POST.get("install_embedded_resources", "false").strip().lower() == "true"
)
if not "file" in request.FILES:
return build_error_response(request, 400, _("Missing component file in the request"))
downloaded_file = request.FILES["file"]
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _("The uploaded file is not a zip file"))
elif content_type == "application/octet-stream":
downloaded_file = BytesIO(request.body)
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _("The uploaded file is not a zip file"))
force_create = request.GET.get("force_create", "false").strip().lower() == "true"
install_embedded_resources = (
request.GET.get("install_embedded_resources", "false").strip().lower() == "true"
)
else:
market_endpoint = None
if content_type == "application/json":
try:
data = json.loads(request.body)
except ValueError as e:
msg = _("malformed json data: %s") % unicode(e)
return build_error_response(request, 400, msg)
install_embedded_resources = normalize_boolean_param(
"install_embedded_resources", data.get("install_embedded_resources", False)
)
force_create = data.get("force_create", False)
templateURL = data.get("template_uri")
market_endpoint = data.get("market_endpoint", None)
else:
force_create = request.POST.get("force_create", False) == "true"
if "url" in request.POST:
templateURL = request.POST["url"]
if market_endpoint is not None:
if "name" not in market_endpoint:
msg = _("Missing market name")
return build_error_response(request, 400, msg)
market_id = market_endpoint["name"]
market_managers = get_market_managers(request.user)
if market_id not in market_managers:
return build_error_response(request, 409, _("Unknown market: %s") % market_id)
market_manager = market_managers[market_id]
downloaded_file = market_manager.download_resource(request.user, templateURL, market_endpoint)
else:
try:
downloaded_file = download_http_content(templateURL)
except:
return build_error_response(request, 409, _("Content cannot be downloaded from the marketplace"))
try:
downloaded_file = BytesIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400, _("The file downloaded from the marketplace is not a zip file")
)
try:
resource = install_resource_to_user(
request.user, file_contents=file_contents, templateURL=templateURL, raise_conflicts=force_create
)
except OSError as e:
if e.errno == errno.EACCES:
return build_error_response(
request,
500,
_("Error writing the resource into the filesystem. Please, contact the server administrator."),
)
else:
raise
except TemplateParseException as e:
msg = "Error parsing config.xml descriptor file: %s" % e
details = "%s" % e
return build_error_response(request, 400, msg, details=details)
except (InvalidContents, UnsupportedFeature) as e:
return build_error_response(request, 400, unicode(e))
except IntegrityError:
return build_error_response(request, 409, _("Resource already exists"))
if install_embedded_resources:
info = {"resource_details": resource.get_processed_info(request), "extra_resources": []}
if resource.resource_type() == "mashup":
resource_info = resource.get_processed_info(process_urls=False)
for embedded_resource in resource_info["embedded"]:
if embedded_resource["src"].startswith("https://"):
resource_file = download_http_content(embedded_resource["src"])
else:
resource_file = BytesIO(file_contents.read(embedded_resource["src"]))
extra_resource_contents = WgtFile(resource_file)
extra_resource = install_resource_to_user(
request.user, file_contents=extra_resource_contents, raise_conflicts=False
)
info["extra_resources"].append(extra_resource.get_processed_info(request))
return HttpResponse(json.dumps(info), status=201, content_type="application/json; charset=UTF-8")
else:
return HttpResponse(
json.dumps(resource.get_processed_info(request)),
status=201,
content_type="application/json; charset=UTF-8",
)
def create(self, request):
status_code = 201
force_create = False
install_embedded_resources = False
templateURL = None
file_contents = None
content_type = get_content_type(request)[0]
if content_type == 'multipart/form-data':
force_create = request.POST.get('force_create', 'false').strip().lower() == 'true'
install_embedded_resources = request.POST.get('install_embedded_resources', 'false').strip().lower() == 'true'
if not 'file' in request.FILES:
return build_error_response(request, 400, _('Missing component file in the request'))
downloaded_file = request.FILES['file']
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The uploaded file is not a zip file'))
elif content_type == 'application/octet-stream':
downloaded_file = BytesIO(request.body)
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The uploaded file is not a zip file'))
force_create = request.GET.get('force_create', 'false').strip().lower() == 'true'
install_embedded_resources = request.GET.get('install_embedded_resources', 'false').strip().lower() == 'true'
else: # if content_type == 'application/json'
market_endpoint = None
data = parse_json_request(request)
install_embedded_resources = normalize_boolean_param(request, 'install_embedded_resources', data.get('install_embedded_resources', False))
force_create = data.get('force_create', False)
templateURL = data.get('url')
market_endpoint = data.get('market_endpoint', None)
if market_endpoint is not None:
if 'name' not in market_endpoint:
msg = _('Missing market name')
return build_error_response(request, 400, msg)
market_id = market_endpoint['name']
market_managers = get_market_managers(request.user)
if market_id not in market_managers:
return build_error_response(request, 409, _('Unknown market: %s') % market_id)
market_manager = market_managers[market_id]
downloaded_file = market_manager.download_resource(request.user, templateURL, market_endpoint)
else:
try:
downloaded_file = download_http_content(templateURL)
except:
return build_error_response(request, 409, _('Content cannot be downloaded from the specified url'))
try:
downloaded_file = BytesIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The file downloaded from the marketplace is not a zip file'))
try:
added, resource = install_resource_to_user(request.user, file_contents=file_contents, templateURL=templateURL)
if not added and force_create:
return build_error_response(request, 409, _('Resource already exists'))
elif not added:
status_code = 200
except zipfile.BadZipfile as e:
return build_error_response(request, 400, _('The uploaded file is not a valid zip file'), details="{}".format(e))
except OSError as e:
if e.errno == errno.EACCES:
return build_error_response(request, 500, _('Error writing the resource into the filesystem. Please, contact the server administrator.'))
else:
raise
except TemplateParseException as e:
msg = "Error parsing config.xml descriptor file: %s" % e
details = "%s" % e
return build_error_response(request, 400, msg, details=details)
except (InvalidContents, UnsupportedFeature) as e:
details = e.details if hasattr(e, 'details') else None
return build_error_response(request, 400, e, details=six.text_type(details))
if install_embedded_resources:
info = {
'resource_details': resource.get_processed_info(request),
'extra_resources': []
}
if resource.resource_type() == 'mashup':
resource_info = resource.get_processed_info(process_urls=False)
for embedded_resource in resource_info['embedded']:
resource_file = BytesIO(file_contents.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
extra_resource_added, extra_resource = install_resource_to_user(request.user, file_contents=extra_resource_contents, raise_conflicts=False)
if extra_resource_added:
info['extra_resources'].append(extra_resource.get_processed_info(request))
return HttpResponse(json.dumps(info), status=status_code, content_type='application/json; charset=UTF-8')
else:
return HttpResponse(json.dumps(resource.get_processed_info(request)), status=status_code, content_type='application/json; charset=UTF-8')
def create(self, request):
status_code = 201
force_create = False
install_embedded_resources = False
templateURL = None
file_contents = None
if request.mimetype == 'multipart/form-data':
force_create = request.POST.get('force_create',
'false').strip().lower() == 'true'
public = request.POST.get('public',
'false').strip().lower() == 'true'
user_list = set(
user.strip()
for user in request.POST.get('users', '').split(',')
if user != "")
group_list = set(
group.strip()
for group in request.POST.get('groups', '').split(',')
if group != "")
install_embedded_resources = request.POST.get(
'install_embedded_resources',
'false').strip().lower() == 'true'
if 'file' not in request.FILES:
return build_error_response(
request, 400, _('Missing component file in the request'))
downloaded_file = request.FILES['file']
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400, _('The uploaded file is not a zip file'))
elif request.mimetype == 'application/octet-stream':
downloaded_file = BytesIO(request.body)
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400, _('The uploaded file is not a zip file'))
force_create = request.GET.get('force_create',
'false').strip().lower() == 'true'
public = request.GET.get('public',
'false').strip().lower() == 'true'
user_list = set(user.strip()
for user in request.GET.get('users', '').split(',')
if user != "")
group_list = set(
group.strip()
for group in request.GET.get('groups', '').split(',')
if group != "")
install_embedded_resources = request.GET.get(
'install_embedded_resources',
'false').strip().lower() == 'true'
else: # if request.mimetype == 'application/json'
market_endpoint = None
data = parse_json_request(request)
install_embedded_resources = normalize_boolean_param(
request, 'install_embedded_resources',
data.get('install_embedded_resources', False))
force_create = data.get('force_create', False)
public = request.GET.get('public',
'false').strip().lower() == 'true'
user_list = set(
user.strip()
for user in request.GET.get('user_list', '').split(',')
if user != "")
group_list = set(
group.strip()
for group in request.GET.get('group_list', '').split(',')
if group != "")
templateURL = data.get('url')
market_endpoint = data.get('market_endpoint', None)
headers = data.get('headers', {})
headers['Accept-Encoding'] = 'identity'
if market_endpoint is not None:
if 'name' not in market_endpoint:
msg = _('Missing market name')
return build_error_response(request, 400, msg)
market_id = market_endpoint['name']
market_managers = get_market_managers(request.user)
if market_id not in market_managers:
return build_error_response(
request, 409,
_('Unknown market: %s') % market_id)
market_manager = market_managers[market_id]
downloaded_file = market_manager.download_resource(
request.user, templateURL, market_endpoint)
else:
try:
context = parse_context_from_referer(request)
except Exception:
context = {}
try:
context["headers"] = CaseInsensitiveDict(headers)
response = WIRECLOUD_PROXY.do_request(
request, templateURL, "GET", context)
if response.status_code >= 300 or response.status_code < 200:
raise Exception()
downloaded_file = b''.join(response)
except Exception:
return build_error_response(
request, 409,
_('Content cannot be downloaded from the specified url'
))
try:
downloaded_file = BytesIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(
request, 400,
_('The file downloaded from the marketplace is not a zip file'
))
if public is False and len(user_list) == 0 and len(group_list) == 0:
users = (request.user, )
else:
users = User.objects.filter(username__in=user_list)
groups = Group.objects.filter(name__in=group_list)
if not request.user.is_superuser:
if public:
return build_error_response(
request, 403,
_('You are not allowed to make resources publicly available to all users'
))
elif len(users) > 0 and tuple(users) != (request.user, ):
return build_error_response(
request, 403,
_('You are not allowed allow to install components to other users'
))
elif len(groups) > 0:
for group in groups:
try:
owners = group.organization.team_set.get(name="owners")
except ObjectDoesNotExist:
fail = True
else:
fail = owners.users.filter(
id=request.user.id).exists() is False
if fail:
return build_error_response(
request, 403,
_('You are not allowed to install components to non-owned organizations'
))
try:
fix_dev_version(file_contents, request.user)
added, resource = install_component(file_contents,
executor_user=request.user,
public=public,
users=users,
groups=groups)
if not added and force_create:
return build_error_response(request, 409,
_('Resource already exists'))
elif not added:
status_code = 200
except zipfile.BadZipfile as e:
return build_error_response(
request,
400,
_('The uploaded file is not a valid zip file'),
details="{}".format(e))
except OSError as e:
if e.errno == errno.EACCES:
return build_error_response(
request, 500,
_('Error writing the resource into the filesystem. Please, contact the server administrator.'
))
else:
raise
except TemplateParseException as e:
msg = "Error parsing config.xml descriptor file: %s" % e
details = "%s" % e
return build_error_response(request, 400, msg, details=details)
except (InvalidContents, UnsupportedFeature) as e:
details = e.details if hasattr(e, 'details') else None
return build_error_response(request, 400, e, details=str(details))
if install_embedded_resources:
info = {
'resource_details':
resource.get_processed_info(
request, url_pattern_name="wirecloud.showcase_media"),
'extra_resources': []
}
if resource.resource_type() == 'mashup':
resource_info = resource.get_processed_info(process_urls=False)
for embedded_resource in resource_info['embedded']:
resource_file = BytesIO(
file_contents.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
extra_resource_added, extra_resource = install_component(
extra_resource_contents,
executor_user=request.user,
public=public,
users=users,
groups=groups)
if extra_resource_added:
info['extra_resources'].append(
extra_resource.get_processed_info(
request,
url_pattern_name="wirecloud.showcase_media"))
response = HttpResponse(
json.dumps(info, sort_keys=True),
status=status_code,
content_type='application/json; charset=UTF-8')
else:
response = HttpResponse(
json.dumps(resource.get_processed_info(
request, url_pattern_name="wirecloud.showcase_media"),
sort_keys=True),
status=status_code,
content_type='application/json; charset=UTF-8')
response['Location'] = resource.get_template_url()
return response
def create(self, request):
status_code = 201
force_create = False
install_embedded_resources = False
templateURL = None
file_contents = None
content_type = get_content_type(request)[0]
if content_type == 'multipart/form-data':
force_create = request.POST.get('force_create', 'false').strip().lower() == 'true'
public = request.POST.get('public', 'false').strip().lower() == 'true'
install_embedded_resources = request.POST.get('install_embedded_resources', 'false').strip().lower() == 'true'
if 'file' not in request.FILES:
return build_error_response(request, 400, _('Missing component file in the request'))
downloaded_file = request.FILES['file']
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The uploaded file is not a zip file'))
elif content_type == 'application/octet-stream':
downloaded_file = BytesIO(request.body)
try:
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The uploaded file is not a zip file'))
force_create = request.GET.get('force_create', 'false').strip().lower() == 'true'
public = request.GET.get('public', 'false').strip().lower() == 'true'
install_embedded_resources = request.GET.get('install_embedded_resources', 'false').strip().lower() == 'true'
else: # if content_type == 'application/json'
market_endpoint = None
data = parse_json_request(request)
install_embedded_resources = normalize_boolean_param(request, 'install_embedded_resources', data.get('install_embedded_resources', False))
force_create = data.get('force_create', False)
public = request.GET.get('public', 'false').strip().lower() == 'true'
templateURL = data.get('url')
market_endpoint = data.get('market_endpoint', None)
headers = data.get('headers', {})
if market_endpoint is not None:
if 'name' not in market_endpoint:
msg = _('Missing market name')
return build_error_response(request, 400, msg)
market_id = market_endpoint['name']
market_managers = get_market_managers(request.user)
if market_id not in market_managers:
return build_error_response(request, 409, _('Unknown market: %s') % market_id)
market_manager = market_managers[market_id]
downloaded_file = market_manager.download_resource(request.user, templateURL, market_endpoint)
else:
try:
context = parse_context_from_referer(request)
except:
context = {}
try:
context["headers"] = CaseInsensitiveDict(headers)
response = WIRECLOUD_PROXY.do_request(request, templateURL, "GET", context)
if response.status_code >= 300 or response.status_code < 200:
raise Exception()
downloaded_file = b''.join(response)
except:
return build_error_response(request, 409, _('Content cannot be downloaded from the specified url'))
try:
downloaded_file = BytesIO(downloaded_file)
file_contents = WgtFile(downloaded_file)
except zipfile.BadZipfile:
return build_error_response(request, 400, _('The file downloaded from the marketplace is not a zip file'))
if public and not request.user.is_superuser:
return build_error_response(request, 403, _('You are not allowed to make resources publicly available to all users'))
try:
fix_dev_version(file_contents, request.user)
added, resource = install_resource_to_user(request.user, file_contents=file_contents, templateURL=templateURL)
if not added and force_create:
return build_error_response(request, 409, _('Resource already exists'))
elif not added:
status_code = 200
if public:
install_resource_to_all_users(executor_user=request.user, file_contents=file_contents)
except zipfile.BadZipfile as e:
return build_error_response(request, 400, _('The uploaded file is not a valid zip file'), details="{}".format(e))
except OSError as e:
if e.errno == errno.EACCES:
return build_error_response(request, 500, _('Error writing the resource into the filesystem. Please, contact the server administrator.'))
else:
raise
except TemplateParseException as e:
msg = "Error parsing config.xml descriptor file: %s" % e
details = "%s" % e
return build_error_response(request, 400, msg, details=details)
except (InvalidContents, UnsupportedFeature) as e:
details = e.details if hasattr(e, 'details') else None
return build_error_response(request, 400, e, details=six.text_type(details))
if install_embedded_resources:
info = {
'resource_details': resource.get_processed_info(request, url_pattern_name="wirecloud.showcase_media"),
'extra_resources': []
}
if resource.resource_type() == 'mashup':
resource_info = resource.get_processed_info(process_urls=False)
for embedded_resource in resource_info['embedded']:
resource_file = BytesIO(file_contents.read(embedded_resource['src']))
extra_resource_contents = WgtFile(resource_file)
if public:
extra_resource_added, extra_resource = install_resource_to_user(request.user, file_contents=extra_resource_contents, raise_conflicts=False)
else:
extra_resource_added, extra_resource = install_resource_to_user(request.user, file_contents=extra_resource_contents, raise_conflicts=False)
if extra_resource_added:
info['extra_resources'].append(extra_resource.get_processed_info(request, url_pattern_name="wirecloud.showcase_media"))
return HttpResponse(json.dumps(info, sort_keys=True), status=status_code, content_type='application/json; charset=UTF-8')
else:
return HttpResponse(json.dumps(resource.get_processed_info(request, url_pattern_name="wirecloud.showcase_media"), sort_keys=True), status=status_code, content_type='application/json; charset=UTF-8')
def create(self, request, workspace_id):
import wirecloud.catalogue.utils as catalogue_utils
content_type = get_content_type(request)[0]
image_file = None
smartphoneimage_file = None
extra_files = []
if content_type == 'application/json':
received_json = request.body.decode('utf-8')
else:
received_json = request.POST['json']
image_file = request.FILES.get('image', None)
smartphoneimage_file = request.FILES.get('smartphoneimage', None)
try:
options = json.loads(received_json)
except ValueError as e:
msg = _("malformed json data: %s") % e
return build_error_response(request, 400, msg)
missing_fields = check_json_fields(options,
('name', 'vendor', 'version'))
if len(missing_fields) > 0:
return build_error_response(
request, 400,
_('Malformed JSON. The following field(s) are missing: %(fields)s.'
) % {'fields': missing_fields})
if not is_valid_vendor(options['vendor']):
return build_error_response(request, 400, _('Invalid vendor'))
if not is_valid_name(options['name']):
return build_error_response(request, 400, _('Invalid name'))
if not is_valid_version(options['version']):
return build_error_response(request, 400,
_('Invalid version number'))
workspace = get_object_or_404(Workspace, id=workspace_id)
if image_file is not None:
options['image'] = 'images/catalogue' + os.path.splitext(
image_file.name)[1]
extra_files.append((options['image'], image_file))
if smartphoneimage_file is not None:
options[
'smartphoneimage'] = 'images/smartphone' + os.path.splitext(
smartphoneimage_file.name)[1]
extra_files.append(
(options['smartphoneimage'], smartphoneimage_file))
if 'longdescription' in options:
extra_files.append(
('DESCRIPTION.md',
BytesIO(options['longdescription'].encode('utf-8'))))
options['longdescription'] = 'DESCRIPTION.md'
description = build_xml_template_from_workspace(
options, workspace, request.user)
# Build mashup wgt file
f = BytesIO()
zf = zipfile.ZipFile(f, 'w')
zf.writestr('config.xml', description.encode('utf-8'))
for filename, extra_file in extra_files:
zf.writestr(filename, extra_file.read())
for resource_info in options['embedded']:
(vendor, name,
version) = (resource_info['vendor'], resource_info['name'],
resource_info['version'])
resource = CatalogueResource.objects.get(vendor=vendor,
short_name=name,
version=version)
base_dir = catalogue_utils.wgt_deployer.get_base_dir(
vendor, name, version)
zf.write(os.path.join(base_dir, resource.template_uri),
resource_info['src'])
zf.close()
wgt_file = WgtFile(f)
resource = get_local_catalogue().publish(None, wgt_file, request.user,
options, request)
return HttpResponse(json.dumps(resource.get_processed_info(request),
ensure_ascii=False),
status=201,
content_type='application/json; charset=utf-8')
def create(self, request):
data = parse_json_request(request)
workspace_name = data.get('name', '').strip()
workspace_title = data.get('title', '').strip()
workspace_id = data.get('workspace', '')
mashup_id = data.get('mashup', '')
initial_pref_values = data.get('preferences', {})
allow_renaming = normalize_boolean_param(
request, 'allow_renaming', data.get('allow_renaming', False))
dry_run = normalize_boolean_param(request, 'dry_run',
data.get('dry_run', False))
if mashup_id == '' and workspace_id == '' and (workspace_name == '' and
workspace_title == ''):
return build_error_response(request, 422,
_('Missing name or title parameter'))
elif mashup_id != '' and workspace_id != '':
return build_error_response(
request, 422,
_('Workspace and mashup parameters cannot be used at the same time'
))
if mashup_id == '' and workspace_id == '':
if workspace_title == '':
workspace_title = workspace_name
if workspace_name != '' and not is_valid_name(workspace_name):
return build_error_response(request, 422,
_('invalid workspace name'))
if dry_run:
return HttpResponse(status=204)
try:
workspace = createEmptyWorkspace(workspace_title,
request.user,
name=workspace_name,
allow_renaming=allow_renaming)
except IntegrityError:
msg = _('A workspace with the given name already exists')
return build_error_response(request, 409, msg)
else:
if mashup_id != '':
values = mashup_id.split('/', 3)
if len(values) != 3:
return build_error_response(request, 422,
_('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(
vendor=mashup_vendor,
short_name=mashup_name,
version=mashup_version)
if not resource.is_available_for(
request.user
) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
return build_error_response(
request, 422,
_('Mashup not found: %(mashup_id)s') %
{'mashup_id': mashup_id})
base_dir = catalogue.wgt_deployer.get_base_dir(
mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(
os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
else:
from_ws = get_object_or_404(Workspace, id=workspace_id)
if from_ws.public is False and not request.user.is_superuser and from_ws.creator != request.user:
return build_error_response(
request, 403,
_('You are not allowed to read from workspace %s') %
workspace_id)
options = {
'vendor':
'api',
'name':
from_ws.name,
'version':
'1.0',
'title':
from_ws.title if from_ws.title is not None
and from_ws.title.strip() != "" else from_ws.name,
'description':
'Temporal mashup for the workspace copy operation',
'email':
'*****@*****.**',
}
template = TemplateParser(
build_json_template_from_workspace(options, from_ws,
from_ws.creator))
try:
check_mashup_dependencies(template, request.user)
except MissingDependencies as e:
details = {
'missingDependencies': e.missing_dependencies,
}
return build_error_response(request, 422, e, details=details)
if dry_run:
return HttpResponse(status=204)
try:
workspace, _junk = buildWorkspaceFromTemplate(
template,
request.user,
allow_renaming=allow_renaming,
new_name=workspace_name,
new_title=workspace_title)
except IntegrityError:
msg = _('A workspace with the given name already exists')
return build_error_response(request, 409, msg)
if len(initial_pref_values) > 0:
update_workspace_preferences(workspace,
initial_pref_values,
invalidate_cache=False)
workspace_data = get_global_workspace_data(workspace, request.user)
return workspace_data.get_response(status_code=201, cacheable=False)
def _handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
users = []
groups = []
redeploy = options['redeploy']
public = options['public']
users_string = options['users'].strip()
groups_string = options['groups'].strip()
if redeploy is False and public is False and users_string == '' and groups_string == '':
raise CommandError(
_('You must use at least one of the following flags: --redeploy, --users, --groups or --public'
))
if not options['redeploy']:
if users_string != '':
for username in users_string.split(','):
users.append(User.objects.get(username=username))
if groups_string != '':
for groupname in groups_string.split(','):
groups.append(Group.objects.get(name=groupname))
for file_name in options['files']:
try:
f = open(file_name, 'rb')
wgt_file = WgtFile(f)
except:
self.log(_('Failed to read from %(file_name)s') %
{'file_name': file_name},
level=1)
continue
try:
template_contents = wgt_file.get_template()
template = TemplateParser(template_contents)
if options['redeploy']:
add_packaged_resource(f,
None,
wgt_file=wgt_file,
template=template,
deploy_only=True)
else:
install_component(wgt_file,
public=public,
users=users,
groups=groups)
wgt_file.close()
f.close()
self.log(_(
'Successfully imported \"%(name)s\" from \"%(file_name)s\"'
) % {
'name':
template.get_resource_processed_info()['title'],
'file_name':
file_name
},
level=1)
except:
self.log(_(
'Failed to import the mashable application component from %(file_name)s'
) % {'file_name': file_name},
level=1)
def process(self, request, to_ws_id):
to_ws = get_object_or_404(Workspace, id=to_ws_id)
if not request.user.is_superuser and to_ws.creator != request.user:
return build_error_response(
request, 403,
_('You are not allowed to update this workspace'))
data = parse_json_request(request)
mashup_id = data.get('mashup', '')
workspace_id = data.get('workspace', '')
if mashup_id == '' and workspace_id == '':
return build_error_response(
request, 422, _('Missing workspace or mashup parameter'))
elif mashup_id != '' and workspace_id != '':
return build_error_response(
request, 422,
_('Workspace and mashup parameters cannot be used at the same time'
))
if mashup_id != '':
values = mashup_id.split('/', 3)
if len(values) != 3:
return build_error_response(request, 422,
_('invalid mashup id'))
(mashup_vendor, mashup_name, mashup_version) = values
try:
resource = CatalogueResource.objects.get(
vendor=mashup_vendor,
short_name=mashup_name,
version=mashup_version)
if not resource.is_available_for(
request.user) or resource.resource_type() != 'mashup':
raise CatalogueResource.DoesNotExist
except CatalogueResource.DoesNotExist:
return build_error_response(
request, 422,
_('Mashup not found: %(mashup_id)s') %
{'mashup_id': mashup_id})
base_dir = catalogue.wgt_deployer.get_base_dir(
mashup_vendor, mashup_name, mashup_version)
wgt_file = WgtFile(os.path.join(base_dir, resource.template_uri))
template = TemplateParser(wgt_file.get_template())
else:
from_ws = get_object_or_404(Workspace, id=workspace_id)
if not request.user.is_superuser and from_ws.creator != request.user:
return build_error_response(
request, 403,
_('You are not allowed to read from workspace %s') %
workspace_id)
options = {
'vendor': 'api',
'name': 'merge_op',
'version': '1.0',
'title': '',
'description': 'Temporal mashup for merging operation',
'email': '*****@*****.**',
}
template = TemplateParser(
build_json_template_from_workspace(options, from_ws,
from_ws.creator))
try:
check_mashup_dependencies(template, request.user)
except MissingDependencies as e:
details = {
'missingDependencies': e.missing_dependencies,
}
return build_error_response(request, 422, e, details=details)
fillWorkspaceUsingTemplate(to_ws, template)
return HttpResponse(status=204)