def do_import_archive(project_id, archive, delete_project=False): project = Project.objects.get(pk=project_id) try: with tempfile.NamedTemporaryFile(suffix='.zip') as archive_file: archive_file.write(archive) archive_file.flush() with zipfile.ZipFile(str(archive_file.name), 'r') as z: contents = z.infolist() # Requirements: # - Find the folder containing the project. This may or may not be at the root level. # - Read in the source files, resources and resource map. # Observations: # - Legal projects must keep their source in a directory called 'src' containing at least one *.c file. # - Legal projects must have a resource map at resources/src/resource_map.json # Strategy: # - Find the shortest common prefix for 'resources/src/resource_map.json' and 'src/'. # - This is taken to be the project directory. # - Import every file in 'src/' with the extension .c or .h as a source file # - Parse resource_map.json and import files it references SRC_DIR = 'src/' WORKER_SRC_DIR = 'worker_src/' INCLUDE_SRC_DIR = 'include/' if len(contents) > 400: raise InvalidProjectArchiveException("Too many files in zip file.") archive_items = [ArchiveProjectItem(z, x) for x in contents] base_dir, manifest_item = find_project_root_and_manifest(archive_items) dir_end = len(base_dir) def make_valid_filename(zip_entry): entry_filename = zip_entry.filename if entry_filename[:dir_end] != base_dir: return False entry_filename = entry_filename[dir_end:] if entry_filename == '': return False if not os.path.normpath('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/%s' % entry_filename).startswith('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/'): raise SuspiciousOperation("Invalid zip file contents.") if zip_entry.file_size > 5242880: # 5 MB raise InvalidProjectArchiveException("Excessively large compressed file.") return entry_filename manifest_kind = make_valid_filename(manifest_item.entry) manifest_dict = json.loads(manifest_item.read()) # Now iterate over the things we found, filter out invalid files and look for the manifest. filtered_contents = [] for entry in contents: filename = make_valid_filename(entry) if not filename or filename in MANIFEST_KINDS: continue else: filtered_contents.append((filename, entry)) with transaction.atomic(): # We have a resource map! We can now try importing things from it. project_options, media_map, dependencies = load_manifest_dict(manifest_dict, manifest_kind) for k, v in project_options.iteritems(): setattr(project, k, v) project.full_clean() project.set_dependencies(dependencies) RES_PATH = project.resources_path tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.iteritems() if v} desired_resources = {} resources_files = {} resource_variants = {} file_exists_for_root = {} # Go through the media map and look for resources for resource in media_map: file_name = resource['file'] identifier = resource['name'] # Pebble.js and simply.js both have some internal resources that we don't import. if project.project_type in {'pebblejs', 'simplyjs'}: if identifier in {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}: continue tags, root_file_name = get_filename_variant(file_name, tag_map) if (len(tags) != 0): raise ValueError("Generic resource filenames cannot contain a tilde (~)") if file_name not in desired_resources: desired_resources[root_file_name] = [] desired_resources[root_file_name].append(resource) file_exists_for_root[root_file_name] = False # Go through the zip file process all resource and source files. for filename, entry in filtered_contents: if filename.startswith(RES_PATH): base_filename = filename[len(RES_PATH) + 1:] # Let's just try opening the file try: extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, base_filename)) except KeyError: logger.debug("Failed to open %s", base_filename) continue # Now we know the file exists and is in the resource directory - is it the one we want? tags, root_file_name = get_filename_variant(base_filename, tag_map) tags_string = ",".join(str(int(t)) for t in tags) if root_file_name in desired_resources: medias = desired_resources[root_file_name] # Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database, # we just use the first one. resource = medias[0] # Make only one resource file per base resource. if root_file_name not in resources_files: kind = resource['type'] is_menu_icon = resource.get('menuIcon', False) resources_files[root_file_name] = ResourceFile.objects.create( project=project, file_name=os.path.basename(root_file_name), kind=kind, is_menu_icon=is_menu_icon) # But add a resource variant for every file actual_file_name = resource['file'] resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string) resource_variants[actual_file_name].save_file(extracted) file_exists_for_root[root_file_name] = True else: try: base_filename, target = SourceFile.get_details_for_path(project.project_type, filename) except ValueError: # We'll just ignore any out of place files. continue source = SourceFile.objects.create(project=project, file_name=base_filename, target=target) with z.open(entry.filename) as f: source.save_text(f.read().decode('utf-8')) # Now add all the resource identifiers for root_file_name in desired_resources: for resource in desired_resources[root_file_name]: target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None ResourceIdentifier.objects.create( resource_file=resources_files[root_file_name], resource_id=resource['name'], target_platforms=target_platforms, # Font options character_regex=resource.get('characterRegex', None), tracking=resource.get('trackingAdjust', None), compatibility=resource.get('compatibility', None), # Bitmap options memory_format=resource.get('memoryFormat', None), storage_format=resource.get('storageFormat', None), space_optimisation=resource.get('spaceOptimization', None) ) # Check that at least one variant of each specified resource exists. for root_file_name, loaded in file_exists_for_root.iteritems(): if not loaded: raise KeyError("No file was found to satisfy the manifest filename: {}".format(root_file_name)) project.save() send_td_event('cloudpebble_zip_import_succeeded', project=project) # At this point we're supposed to have successfully created the project. return True except Exception as e: if delete_project: try: Project.objects.get(pk=project_id).delete() except: pass send_td_event('cloudpebble_zip_import_failed', data={ 'data': { 'reason': str(e) } }, user=project.owner) raise
def import_gist(user_id, gist_id): user = User.objects.get(pk=user_id) g = github.Github() try: gist = g.get_gist(gist_id) except github.UnknownObjectException: send_td_event('cloudpebble_gist_not_found', data={'data': {'gist_id': gist_id}}, user=user) raise Exception("Couldn't find gist to import.") files = gist.files default_name = gist.description or 'Sample project' default_settings = { 'name': default_name, 'app_short_name': default_name, 'app_long_name': default_name, 'app_company_name': user.username, 'app_version_label': '1.0', 'app_is_watchface': False, 'app_is_hidden': False, 'app_is_shown_on_communication': False, 'app_capabilities': '[]', 'app_keys': '{}', 'project_type': 'native', 'app_modern_multi_js': False, 'sdk_version': '2' } if len(files) == 1 or ((APPINFO_MANIFEST in files or PACKAGE_MANIFEST in files) and len(files) == 2): if 'simply.js' in files: default_settings['project_type'] = 'simplyjs' elif 'app.js' in files: default_settings['project_type'] = 'pebblejs' elif 'index.js' in files: default_settings['project_type'] = 'rocky' # If all files are .js or .json and there is an index.js, assume it's a rocky project. if all(x.endswith(('.js', '.json')) for x in gist.files) and 'index.js' in files: default_settings['project_type'] = 'rocky' default_settings['sdk_version'] = '3' default_settings['app_modern_multi_js'] = True media = [] # Using defaultdict we can load project settings from a manifest dict which # has values that default to None. This way, we can delegate if PACKAGE_MANIFEST in files: content = json.loads(files[PACKAGE_MANIFEST].content) package = defaultdict(lambda: None) package.update(content) package['pebble'] = defaultdict(lambda: None) package['pebble'].update(content.get('pebble', {})) manifest_settings, media, dependencies = load_manifest_dict(package, PACKAGE_MANIFEST, default_project_type=None) default_settings['app_keys'] = '[]' default_settings['sdk_version'] = '3' default_settings['app_modern_multi_js'] = True elif APPINFO_MANIFEST in files: content = json.loads(files[APPINFO_MANIFEST].content) package = defaultdict(lambda: None) package.update(content) manifest_settings, media, dependencies = load_manifest_dict(package, APPINFO_MANIFEST, default_project_type=None) else: manifest_settings = {} dependencies = {} fixed_settings = { 'owner': user, 'app_uuid': generate_half_uuid() } project_settings = {} project_settings.update(default_settings) project_settings.update({k: v for k, v in manifest_settings.iteritems() if v is not None}) project_settings.update(fixed_settings) with transaction.atomic(): project = Project.objects.create(**project_settings) project.set_dependencies(dependencies) project_type = project.project_type if project_type == 'package': raise Exception("Gist imports are not yet support for packages.") if project_type != 'simplyjs': for filename in gist.files: target = 'app' if not filename.endswith(('.c', '.h', '.js', '.json')): continue if filename in ('appinfo.json', 'package.json'): continue if project_type == 'native': if filename.endswith(('.js', '.json')): target = 'pkjs' elif project_type == 'rocky': if filename == 'app.js': target = 'pkjs' source_file = SourceFile.objects.create(project=project, file_name=filename, target=target) source_file.save_text(gist.files[filename].content) resources = {} for resource in media: kind = resource['type'] def_name = resource['name'] filename = resource['file'] regex = resource.get('characterRegex', None) tracking = resource.get('trackingAdjust', None) memory_format = resource.get('memoryFormat', None) storage_format = resource.get('storageFormat', None) space_optimisation = resource.get('spaceOptimization', None) is_menu_icon = resource.get('menuIcon', False) compatibility = resource.get('compatibility', None) if filename not in gist.files: continue if filename not in resources: resources[filename] = ResourceFile.objects.create(project=project, file_name=filename, kind=kind, is_menu_icon=is_menu_icon) # We already have this as a unicode string in .content, but it shouldn't have become unicode # in the first place. default_variant = ResourceVariant.objects.create(resource_file=resources[filename], tags=ResourceVariant.TAGS_DEFAULT) default_variant.save_file(urllib2.urlopen(gist.files[filename].raw_url)) ResourceIdentifier.objects.create( resource_file=resources[filename], resource_id=def_name, character_regex=regex, tracking=tracking, compatibility=compatibility, memory_format=memory_format, storage_format=storage_format, space_optimisation=space_optimisation ) else: source_file = SourceFile.objects.create(project=project, file_name='app.js') source_file.save_text(gist.files['simply.js'].content) send_td_event('cloudpebble_gist_import', data={'data': {'gist_id': gist_id}}, project=project) return project.id
def import_gist(user_id, gist_id): user = User.objects.get(pk=user_id) g = github.Github() try: gist = g.get_gist(gist_id) except github.UnknownObjectException: send_td_event('cloudpebble_gist_not_found', data={'data': {'gist_id': gist_id}}, user=user) raise Exception("Couldn't find gist to import.") files = gist.files default_name = gist.description or 'Sample project' project_type = 'native' default_settings = { 'name': default_name, 'app_short_name': default_name, 'app_long_name': default_name, 'app_company_name': user.username, 'app_version_label': '1.0', 'app_is_watchface': False, 'app_is_hidden': False, 'app_is_shown_on_communication': False, 'app_capabilities': '[]', 'app_keys': '{}', 'project_type': 'native', 'app_modern_multi_js': False, 'sdk_version': '2' } if len(files) == 1 or ((APPINFO_MANIFEST in files or PACKAGE_MANIFEST in files) and len(files) == 2): if 'simply.js' in files: default_settings['project_type'] = 'simplyjs' elif 'app.js' in files: default_settings['project_type'] = 'pebblejs' media = [] # Using defaultdict we can load project settings from a manifest dict which # has values that default to None. This way, we can delegate if PACKAGE_MANIFEST in files: content = json.loads(files[PACKAGE_MANIFEST].content) package = defaultdict(lambda: None) package.update(content) package['pebble'] = defaultdict(lambda: None) package['pebble'].update(content.get('pebble', {})) manifest_settings, media, dependencies = load_manifest_dict(package, PACKAGE_MANIFEST, default_project_type=None) if settings.NPM_MANIFEST_SUPPORT: default_settings['app_keys'] = '[]' elif APPINFO_MANIFEST in files: content = json.loads(files['appinfo.json'].content) package = defaultdict(lambda: None) package.update(content) manifest_settings, media, dependencies = load_manifest_dict(package, APPINFO_MANIFEST, default_project_type=None) else: manifest_settings = {} dependencies = {} fixed_settings = { 'owner': user, 'app_uuid': generate_half_uuid() } project_settings = {} project_settings.update(default_settings) project_settings.update({k: v for k, v in manifest_settings.iteritems() if v is not None}) project_settings.update(fixed_settings) with transaction.atomic(): project = Project.objects.create(**project_settings) project.set_dependencies(dependencies) if project_type != 'simplyjs': for filename in gist.files: if (project_type == 'native' and filename.endswith('.c') or filename.endswith('.h')) or filename.endswith('.js'): # Because gists can't have subdirectories. if filename == 'pebble-js-app.js': cp_filename = 'js/pebble-js-app.js' else: cp_filename = filename source_file = SourceFile.objects.create(project=project, file_name=cp_filename) source_file.save_file(gist.files[filename].content) resources = {} for resource in media: kind = resource['type'] def_name = resource['name'] filename = resource['file'] regex = resource.get('characterRegex', None) tracking = resource.get('trackingAdjust', None) memory_format = resource.get('memoryFormat', None) storage_format = resource.get('storageFormat', None) space_optimisation = resource.get('spaceOptimization', None) is_menu_icon = resource.get('menuIcon', False) compatibility = resource.get('compatibility', None) if filename not in gist.files: continue if filename not in resources: resources[filename] = ResourceFile.objects.create(project=project, file_name=filename, kind=kind, is_menu_icon=is_menu_icon) # We already have this as a unicode string in .content, but it shouldn't have become unicode # in the first place. default_variant = ResourceVariant.objects.create(resource_file=resources[filename], tags=ResourceVariant.TAGS_DEFAULT) default_variant.save_file(urllib2.urlopen(gist.files[filename].raw_url)) ResourceIdentifier.objects.create( resource_file=resources[filename], resource_id=def_name, character_regex=regex, tracking=tracking, compatibility=compatibility, memory_format=memory_format, storage_format=storage_format, space_optimisation=space_optimisation ) else: source_file = SourceFile.objects.create(project=project, file_name='app.js') source_file.save_file(gist.files['simply.js'].content) send_td_event('cloudpebble_gist_import', data={'data': {'gist_id': gist_id}}, project=project) return project.id