def do_import_archive(project_id, archive, delete_project=False):
project = Project.objects.get(pk=project_id)
try:
with tempfile.NamedTemporaryFile(suffix='.zip') as archive_file:
archive_file.write(archive)
archive_file.flush()
with zipfile.ZipFile(str(archive_file.name), 'r') as z:
contents = z.infolist()
# Requirements:
# - Find the folder containing the project. This may or may not be at the root level.
# - Read in the source files, resources and resource map.
# Observations:
# - Legal projects must keep their source in a directory called 'src' containing at least one *.c file.
# - Legal projects must have a resource map at resources/src/resource_map.json
# Strategy:
# - Find the shortest common prefix for 'resources/src/resource_map.json' and 'src/'.
# - This is taken to be the project directory.
# - Import every file in 'src/' with the extension .c or .h as a source file
# - Parse resource_map.json and import files it references
SRC_DIR = 'src/'
WORKER_SRC_DIR = 'worker_src/'
INCLUDE_SRC_DIR = 'include/'
if len(contents) > 400:
raise InvalidProjectArchiveException("Too many files in zip file.")
archive_items = [ArchiveProjectItem(z, x) for x in contents]
base_dir, manifest_item = find_project_root_and_manifest(archive_items)
dir_end = len(base_dir)
def make_valid_filename(zip_entry):
entry_filename = zip_entry.filename
if entry_filename[:dir_end] != base_dir:
return False
entry_filename = entry_filename[dir_end:]
if entry_filename == '':
return False
if not os.path.normpath('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/%s' % entry_filename).startswith('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/'):
raise SuspiciousOperation("Invalid zip file contents.")
if zip_entry.file_size > 5242880: # 5 MB
raise InvalidProjectArchiveException("Excessively large compressed file.")
return entry_filename
manifest_kind = make_valid_filename(manifest_item.entry)
manifest_dict = json.loads(manifest_item.read())
# Now iterate over the things we found, filter out invalid files and look for the manifest.
filtered_contents = []
for entry in contents:
filename = make_valid_filename(entry)
if not filename or filename in MANIFEST_KINDS:
continue
else:
filtered_contents.append((filename, entry))
with transaction.atomic():
# We have a resource map! We can now try importing things from it.
project_options, media_map, dependencies = load_manifest_dict(manifest_dict, manifest_kind)
for k, v in project_options.iteritems():
setattr(project, k, v)
project.full_clean()
project.set_dependencies(dependencies)
RES_PATH = project.resources_path
tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.iteritems() if v}
desired_resources = {}
resources_files = {}
resource_variants = {}
file_exists_for_root = {}
# Go through the media map and look for resources
for resource in media_map:
file_name = resource['file']
identifier = resource['name']
# Pebble.js and simply.js both have some internal resources that we don't import.
if project.project_type in {'pebblejs', 'simplyjs'}:
if identifier in {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}:
continue
tags, root_file_name = get_filename_variant(file_name, tag_map)
if (len(tags) != 0):
raise ValueError("Generic resource filenames cannot contain a tilde (~)")
if file_name not in desired_resources:
desired_resources[root_file_name] = []
desired_resources[root_file_name].append(resource)
file_exists_for_root[root_file_name] = False
# Go through the zip file process all resource and source files.
for filename, entry in filtered_contents:
if filename.startswith(RES_PATH):
base_filename = filename[len(RES_PATH) + 1:]
# Let's just try opening the file
try:
extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, base_filename))
except KeyError:
logger.debug("Failed to open %s", base_filename)
continue
# Now we know the file exists and is in the resource directory - is it the one we want?
tags, root_file_name = get_filename_variant(base_filename, tag_map)
tags_string = ",".join(str(int(t)) for t in tags)
if root_file_name in desired_resources:
medias = desired_resources[root_file_name]
# Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database,
# we just use the first one.
resource = medias[0]
# Make only one resource file per base resource.
if root_file_name not in resources_files:
kind = resource['type']
is_menu_icon = resource.get('menuIcon', False)
resources_files[root_file_name] = ResourceFile.objects.create(
project=project,
file_name=os.path.basename(root_file_name),
kind=kind,
is_menu_icon=is_menu_icon)
# But add a resource variant for every file
actual_file_name = resource['file']
resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string)
resource_variants[actual_file_name].save_file(extracted)
file_exists_for_root[root_file_name] = True
else:
try:
base_filename, target = SourceFile.get_details_for_path(project.project_type, filename)
except ValueError:
# We'll just ignore any out of place files.
continue
source = SourceFile.objects.create(project=project, file_name=base_filename, target=target)
with z.open(entry.filename) as f:
source.save_text(f.read().decode('utf-8'))
# Now add all the resource identifiers
for root_file_name in desired_resources:
for resource in desired_resources[root_file_name]:
target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None
ResourceIdentifier.objects.create(
resource_file=resources_files[root_file_name],
resource_id=resource['name'],
target_platforms=target_platforms,
# Font options
character_regex=resource.get('characterRegex', None),
tracking=resource.get('trackingAdjust', None),
compatibility=resource.get('compatibility', None),
# Bitmap options
memory_format=resource.get('memoryFormat', None),
storage_format=resource.get('storageFormat', None),
space_optimisation=resource.get('spaceOptimization', None)
)
# Check that at least one variant of each specified resource exists.
for root_file_name, loaded in file_exists_for_root.iteritems():
if not loaded:
raise KeyError("No file was found to satisfy the manifest filename: {}".format(root_file_name))
project.save()
send_td_event('cloudpebble_zip_import_succeeded', project=project)
# At this point we're supposed to have successfully created the project.
return True
except Exception as e:
if delete_project:
try:
Project.objects.get(pk=project_id).delete()
except:
pass
send_td_event('cloudpebble_zip_import_failed', data={
'data': {
'reason': str(e)
}
}, user=project.owner)
raise
def import_gist(user_id, gist_id):
user = User.objects.get(pk=user_id)
g = github.Github()
try:
gist = g.get_gist(gist_id)
except github.UnknownObjectException:
send_td_event('cloudpebble_gist_not_found', data={'data': {'gist_id': gist_id}}, user=user)
raise Exception("Couldn't find gist to import.")
files = gist.files
default_name = gist.description or 'Sample project'
default_settings = {
'name': default_name,
'app_short_name': default_name,
'app_long_name': default_name,
'app_company_name': user.username,
'app_version_label': '1.0',
'app_is_watchface': False,
'app_is_hidden': False,
'app_is_shown_on_communication': False,
'app_capabilities': '[]',
'app_keys': '{}',
'project_type': 'native',
'app_modern_multi_js': False,
'sdk_version': '2'
}
if len(files) == 1 or ((APPINFO_MANIFEST in files or PACKAGE_MANIFEST in files) and len(files) == 2):
if 'simply.js' in files:
default_settings['project_type'] = 'simplyjs'
elif 'app.js' in files:
default_settings['project_type'] = 'pebblejs'
elif 'index.js' in files:
default_settings['project_type'] = 'rocky'
# If all files are .js or .json and there is an index.js, assume it's a rocky project.
if all(x.endswith(('.js', '.json')) for x in gist.files) and 'index.js' in files:
default_settings['project_type'] = 'rocky'
default_settings['sdk_version'] = '3'
default_settings['app_modern_multi_js'] = True
media = []
# Using defaultdict we can load project settings from a manifest dict which
# has values that default to None. This way, we can delegate
if PACKAGE_MANIFEST in files:
content = json.loads(files[PACKAGE_MANIFEST].content)
package = defaultdict(lambda: None)
package.update(content)
package['pebble'] = defaultdict(lambda: None)
package['pebble'].update(content.get('pebble', {}))
manifest_settings, media, dependencies = load_manifest_dict(package, PACKAGE_MANIFEST, default_project_type=None)
default_settings['app_keys'] = '[]'
default_settings['sdk_version'] = '3'
default_settings['app_modern_multi_js'] = True
elif APPINFO_MANIFEST in files:
content = json.loads(files[APPINFO_MANIFEST].content)
package = defaultdict(lambda: None)
package.update(content)
manifest_settings, media, dependencies = load_manifest_dict(package, APPINFO_MANIFEST, default_project_type=None)
else:
manifest_settings = {}
dependencies = {}
fixed_settings = {
'owner': user,
'app_uuid': generate_half_uuid()
}
project_settings = {}
project_settings.update(default_settings)
project_settings.update({k: v for k, v in manifest_settings.iteritems() if v is not None})
project_settings.update(fixed_settings)
with transaction.atomic():
project = Project.objects.create(**project_settings)
project.set_dependencies(dependencies)
project_type = project.project_type
if project_type == 'package':
raise Exception("Gist imports are not yet support for packages.")
if project_type != 'simplyjs':
for filename in gist.files:
target = 'app'
if not filename.endswith(('.c', '.h', '.js', '.json')):
continue
if filename in ('appinfo.json', 'package.json'):
continue
if project_type == 'native':
if filename.endswith(('.js', '.json')):
target = 'pkjs'
elif project_type == 'rocky':
if filename == 'app.js':
target = 'pkjs'
source_file = SourceFile.objects.create(project=project, file_name=filename, target=target)
source_file.save_text(gist.files[filename].content)
resources = {}
for resource in media:
kind = resource['type']
def_name = resource['name']
filename = resource['file']
regex = resource.get('characterRegex', None)
tracking = resource.get('trackingAdjust', None)
memory_format = resource.get('memoryFormat', None)
storage_format = resource.get('storageFormat', None)
space_optimisation = resource.get('spaceOptimization', None)
is_menu_icon = resource.get('menuIcon', False)
compatibility = resource.get('compatibility', None)
if filename not in gist.files:
continue
if filename not in resources:
resources[filename] = ResourceFile.objects.create(project=project, file_name=filename, kind=kind,
is_menu_icon=is_menu_icon)
# We already have this as a unicode string in .content, but it shouldn't have become unicode
# in the first place.
default_variant = ResourceVariant.objects.create(resource_file=resources[filename], tags=ResourceVariant.TAGS_DEFAULT)
default_variant.save_file(urllib2.urlopen(gist.files[filename].raw_url))
ResourceIdentifier.objects.create(
resource_file=resources[filename],
resource_id=def_name,
character_regex=regex,
tracking=tracking,
compatibility=compatibility,
memory_format=memory_format,
storage_format=storage_format,
space_optimisation=space_optimisation
)
else:
source_file = SourceFile.objects.create(project=project, file_name='app.js')
source_file.save_text(gist.files['simply.js'].content)
send_td_event('cloudpebble_gist_import', data={'data': {'gist_id': gist_id}}, project=project)
return project.id
def import_gist(user_id, gist_id):
user = User.objects.get(pk=user_id)
g = github.Github()
try:
gist = g.get_gist(gist_id)
except github.UnknownObjectException:
send_td_event('cloudpebble_gist_not_found', data={'data': {'gist_id': gist_id}}, user=user)
raise Exception("Couldn't find gist to import.")
files = gist.files
default_name = gist.description or 'Sample project'
project_type = 'native'
default_settings = {
'name': default_name,
'app_short_name': default_name,
'app_long_name': default_name,
'app_company_name': user.username,
'app_version_label': '1.0',
'app_is_watchface': False,
'app_is_hidden': False,
'app_is_shown_on_communication': False,
'app_capabilities': '[]',
'app_keys': '{}',
'project_type': 'native',
'app_modern_multi_js': False,
'sdk_version': '2'
}
if len(files) == 1 or ((APPINFO_MANIFEST in files or PACKAGE_MANIFEST in files) and len(files) == 2):
if 'simply.js' in files:
default_settings['project_type'] = 'simplyjs'
elif 'app.js' in files:
default_settings['project_type'] = 'pebblejs'
media = []
# Using defaultdict we can load project settings from a manifest dict which
# has values that default to None. This way, we can delegate
if PACKAGE_MANIFEST in files:
content = json.loads(files[PACKAGE_MANIFEST].content)
package = defaultdict(lambda: None)
package.update(content)
package['pebble'] = defaultdict(lambda: None)
package['pebble'].update(content.get('pebble', {}))
manifest_settings, media, dependencies = load_manifest_dict(package, PACKAGE_MANIFEST, default_project_type=None)
if settings.NPM_MANIFEST_SUPPORT:
default_settings['app_keys'] = '[]'
elif APPINFO_MANIFEST in files:
content = json.loads(files['appinfo.json'].content)
package = defaultdict(lambda: None)
package.update(content)
manifest_settings, media, dependencies = load_manifest_dict(package, APPINFO_MANIFEST, default_project_type=None)
else:
manifest_settings = {}
dependencies = {}
fixed_settings = {
'owner': user,
'app_uuid': generate_half_uuid()
}
project_settings = {}
project_settings.update(default_settings)
project_settings.update({k: v for k, v in manifest_settings.iteritems() if v is not None})
project_settings.update(fixed_settings)
with transaction.atomic():
project = Project.objects.create(**project_settings)
project.set_dependencies(dependencies)
if project_type != 'simplyjs':
for filename in gist.files:
if (project_type == 'native' and filename.endswith('.c') or filename.endswith('.h')) or filename.endswith('.js'):
# Because gists can't have subdirectories.
if filename == 'pebble-js-app.js':
cp_filename = 'js/pebble-js-app.js'
else:
cp_filename = filename
source_file = SourceFile.objects.create(project=project, file_name=cp_filename)
source_file.save_file(gist.files[filename].content)
resources = {}
for resource in media:
kind = resource['type']
def_name = resource['name']
filename = resource['file']
regex = resource.get('characterRegex', None)
tracking = resource.get('trackingAdjust', None)
memory_format = resource.get('memoryFormat', None)
storage_format = resource.get('storageFormat', None)
space_optimisation = resource.get('spaceOptimization', None)
is_menu_icon = resource.get('menuIcon', False)
compatibility = resource.get('compatibility', None)
if filename not in gist.files:
continue
if filename not in resources:
resources[filename] = ResourceFile.objects.create(project=project, file_name=filename, kind=kind,
is_menu_icon=is_menu_icon)
# We already have this as a unicode string in .content, but it shouldn't have become unicode
# in the first place.
default_variant = ResourceVariant.objects.create(resource_file=resources[filename], tags=ResourceVariant.TAGS_DEFAULT)
default_variant.save_file(urllib2.urlopen(gist.files[filename].raw_url))
ResourceIdentifier.objects.create(
resource_file=resources[filename],
resource_id=def_name,
character_regex=regex,
tracking=tracking,
compatibility=compatibility,
memory_format=memory_format,
storage_format=storage_format,
space_optimisation=space_optimisation
)
else:
source_file = SourceFile.objects.create(project=project, file_name='app.js')
source_file.save_file(gist.files['simply.js'].content)
send_td_event('cloudpebble_gist_import', data={'data': {'gist_id': gist_id}}, project=project)
return project.id
def do_import_archive(project_id, archive, delete_project=False):
project = Project.objects.get(pk=project_id)
try:
with tempfile.NamedTemporaryFile(suffix='.zip') as archive_file:
archive_file.write(archive)
archive_file.flush()
with zipfile.ZipFile(str(archive_file.name), 'r') as z:
contents = z.infolist()
# Requirements:
# - Find the folder containing the project. This may or may not be at the root level.
# - Read in the source files, resources and resource map.
# Observations:
# - Legal projects must keep their source in a directory called 'src' containing at least one *.c file.
# - Legal projects must have a resource map at resources/src/resource_map.json
# Strategy:
# - Find the shortest common prefix for 'resources/src/resource_map.json' and 'src/'.
# - This is taken to be the project directory.
# - Import every file in 'src/' with the extension .c or .h as a source file
# - Parse resource_map.json and import files it references
SRC_DIR = 'src/'
WORKER_SRC_DIR = 'worker_src/'
INCLUDE_SRC_DIR = 'include/'
if len(contents) > 400:
raise InvalidProjectArchiveException("Too many files in zip file.")
archive_items = [ArchiveProjectItem(z, x) for x in contents]
base_dir, manifest_item = find_project_root_and_manifest(archive_items)
dir_end = len(base_dir)
def make_valid_filename(zip_entry):
entry_filename = zip_entry.filename
if entry_filename[:dir_end] != base_dir:
return False
entry_filename = entry_filename[dir_end:]
if entry_filename == '':
return False
if not os.path.normpath('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/%s' % entry_filename).startswith('/SENTINEL_DO_NOT_ACTUALLY_USE_THIS_NAME/'):
raise SuspiciousOperation("Invalid zip file contents.")
if zip_entry.file_size > 5242880: # 5 MB
raise InvalidProjectArchiveException("Excessively large compressed file.")
return entry_filename
manifest_kind = make_valid_filename(manifest_item.entry)
manifest_dict = json.loads(manifest_item.read())
# Now iterate over the things we found, filter out invalid files and look for the manifest.
filtered_contents = []
for entry in contents:
filename = make_valid_filename(entry)
if not filename or filename in MANIFEST_KINDS:
continue
else:
filtered_contents.append((filename, entry))
with transaction.atomic():
# We have a resource map! We can now try importing things from it.
project_options, media_map, dependencies = load_manifest_dict(manifest_dict, manifest_kind)
for k, v in project_options.iteritems():
setattr(project, k, v)
project.full_clean()
project.set_dependencies(dependencies)
RES_PATH = project.resources_path
tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.iteritems() if v}
desired_resources = {}
resources_files = {}
resource_variants = {}
file_exists_for_root = {}
# Go through the media map and look for resources
for resource in media_map:
file_name = resource['file']
identifier = resource['name']
# Pebble.js and simply.js both have some internal resources that we don't import.
if project.project_type in {'pebblejs', 'simplyjs'}:
if identifier in {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}:
continue
tags, root_file_name = get_filename_variant(file_name, tag_map)
if (len(tags) != 0):
raise ValueError("Generic resource filenames cannot contain a tilde (~)")
if file_name not in desired_resources:
desired_resources[root_file_name] = []
desired_resources[root_file_name].append(resource)
file_exists_for_root[root_file_name] = False
# Go through the zip file process all resource and source files.
for filename, entry in filtered_contents:
if filename.startswith(RES_PATH):
base_filename = filename[len(RES_PATH) + 1:]
# Let's just try opening the file
try:
extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, base_filename))
except KeyError:
logger.debug("Failed to open %s", base_filename)
continue
# Now we know the file exists and is in the resource directory - is it the one we want?
tags, root_file_name = get_filename_variant(base_filename, tag_map)
tags_string = ",".join(str(int(t)) for t in tags)
if root_file_name in desired_resources:
medias = desired_resources[root_file_name]
# Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database,
# we just use the first one.
resource = medias[0]
# Make only one resource file per base resource.
if root_file_name not in resources_files:
kind = resource['type']
is_menu_icon = resource.get('menuIcon', False)
resources_files[root_file_name] = ResourceFile.objects.create(
project=project,
file_name=os.path.basename(root_file_name),
kind=kind,
is_menu_icon=is_menu_icon)
# But add a resource variant for every file
actual_file_name = resource['file']
resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string)
resource_variants[actual_file_name].save_file(extracted)
file_exists_for_root[root_file_name] = True
else:
try:
base_filename, target = SourceFile.get_details_for_path(project.project_type, filename)
except ValueError:
# We'll just ignore any out of place files.
continue
source = SourceFile.objects.create(project=project, file_name=base_filename, target=target)
with z.open(entry.filename) as f:
source.save_text(f.read().decode('utf-8'))
# Now add all the resource identifiers
for root_file_name in desired_resources:
for resource in desired_resources[root_file_name]:
target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None
ResourceIdentifier.objects.create(
resource_file=resources_files[root_file_name],
resource_id=resource['name'],
target_platforms=target_platforms,
# Font options
character_regex=resource.get('characterRegex', None),
tracking=resource.get('trackingAdjust', None),
compatibility=resource.get('compatibility', None),
# Bitmap options
memory_format=resource.get('memoryFormat', None),
storage_format=resource.get('storageFormat', None),
space_optimisation=resource.get('spaceOptimization', None)
)
# Check that at least one variant of each specified resource exists.
for root_file_name, loaded in file_exists_for_root.iteritems():
if not loaded:
raise KeyError("No file was found to satisfy the manifest filename: {}".format(root_file_name))
project.save()
send_td_event('cloudpebble_zip_import_succeeded', project=project)
# At this point we're supposed to have successfully created the project.
return True
except Exception as e:
if delete_project:
try:
Project.objects.get(pk=project_id).delete()
except:
pass
send_td_event('cloudpebble_zip_import_failed', data={
'data': {
'reason': str(e)
}
}, user=project.owner)
raise