def test_package_manifest_with_dependencies(self): """ Check that dependencies are represented in the package.json file. """ deps = {'some_package': '1.2.3', 'another': '^4.2.0'} self.project.set_dependencies(deps) manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={'dependencies': deps})
def add_project_to_archive(z, project, prefix=''): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r'[^\w]+', '_', project.name).strip('_').lower() for source in source_files: src_dir = 'src' if project.project_type == 'native': if source.target == 'worker': src_dir = 'worker_src' elif project.app_modern_multi_js and source.file_name.endswith( '.js'): src_dir = 'src/js' z.writestr('%s/%s/%s' % (prefix, src_dir, source.file_name), source.get_contents()) for resource in resources: res_path = 'resources' for variant in resource.variants.all(): z.writestr('%s/%s/%s' % (prefix, res_path, variant.path), variant.get_contents()) manifest = generate_manifest(project, resources) manifest_name = manifest_name_for_project(project) z.writestr('%s/%s' % (prefix, manifest_name), manifest) if project.project_type == 'native': # This file is always the same, but needed to build. z.writestr('%s/wscript' % prefix, generate_wscript_file(project, for_export=True)) z.writestr('%s/jshintrc' % prefix, generate_jshint_file(project))
def test_package_manifest_with_keywords(self): """ Check that saved keywords are present in the package.json file """ self.maxDiff = None keywords = ["pebbles...", "are?!", "~{cool}~"] self.project.keywords = keywords manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={"keywords": keywords})
def test_manifest_short_name(self): """ Check that app_short_name is mangled into a valid npm package name """ self.project.app_short_name = "_CAPITALS_and...dots-and - spaces ! " manifest = generate_manifest(self.project, []) self.check_package_manifest( manifest, package_options={'name': 'capitals_and...dots-and-spaces'})
def add_project_to_archive(z, project, prefix=''): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r'[^\w]+', '_', project.name).strip('_').lower() for source in source_files: src_dir = 'src' if project.project_type == 'native': if source.target == 'worker': src_dir = 'worker_src' elif project.app_modern_multi_js and source.file_name.endswith('.js'): src_dir = 'src/js' z.writestr('%s/%s/%s' % (prefix, src_dir, source.file_name), source.get_contents()) for resource in resources: res_path = 'resources' for variant in resource.variants.all(): z.writestr('%s/%s/%s' % (prefix, res_path, variant.path), variant.get_contents()) manifest = generate_manifest(project, resources) z.writestr('%s/appinfo.json' % prefix, manifest) if project.project_type == 'native': # This file is always the same, but needed to build. z.writestr('%s/wscript' % prefix, generate_wscript_file(project, for_export=True)) z.writestr('%s/jshintrc' % prefix, generate_jshint_file(project))
def test_package_manifest_with_keywords(self): """ Check that saved keywords are present in the package.json file """ self.maxDiff = None keywords = ["pebbles...", "are?!", "~{cool}~"] self.project.keywords = keywords manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={'keywords': keywords})
def test_package_manifest_with_dependencies(self): """ Check that dependencies are represented in the package.json file. """ deps = { 'some_package': '1.2.3', 'another': '^4.2.0' } self.project.set_dependencies(deps) manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={'dependencies': deps})
def test_inter_project_dependencies(self): """ Check that inter-project dependencies are represented in the manifest """ package = Project.objects.create( project_type="package", name="test_package", app_short_name="mylib", sdk_version="3", owner_id=self.user_id ) build = BuildResult.objects.create(project=package, state=BuildResult.STATE_SUCCEEDED) self.project.project_dependencies.add(package) deps = {"mylib": build.package_url} manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={"dependencies": deps})
def test_inter_project_dependencies(self): """ Check that inter-project dependencies are represented in the manifest """ package = Project.objects.create(project_type='package', name='test_package', app_short_name='mylib', sdk_version='3', owner_id=self.user_id) build = BuildResult.objects.create(project=package, state=BuildResult.STATE_SUCCEEDED) self.project.project_dependencies.add(package) deps = {'mylib': build.package_url} manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={'dependencies': deps})
def test_published_media(self): """ Check that publishedMedia are represented in the manifest """ published_media = [{ 'name': 'TEST', 'id': 0, 'glance': 'TINY', 'timeline': { 'tiny': 'TINY', 'small': 'SMALL', 'large': 'LARGE', } }] self.project.set_published_media(published_media) manifest = generate_manifest(self.project, []) self.check_package_manifest( manifest, pebble_options={'publishedMedia': published_media})
def add_project_to_archive(z, project, prefix=''): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r'[^\w]+', '_', project.name).strip('_').lower() for source in source_files: src_dir = 'worker_src' if source.target == 'worker' else 'src' z.writestr('%s/%s/%s' % (prefix, src_dir, source.file_name), source.get_contents()) for resource in resources: res_path = 'resources' z.writestr('%s/%s/%s' % (prefix, res_path, resource.path), resource.get_contents()) manifest = generate_manifest(project, resources) z.writestr('%s/appinfo.json' % prefix, manifest) if project.project_type == 'native': # This file is always the same, but needed to build. z.writestr('%s/wscript' % prefix, generate_wscript_file(project, for_export=True)) z.writestr('%s/jshintrc' % prefix, generate_jshint_file(project))
def add_project_to_archive(z, project, prefix=''): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r'[^\w]+', '_', project.name).strip('_').lower() for source in source_files: path = os.path.join(prefix, source.project_path) z.writestr(path, source.get_contents()) for resource in resources: for variant in resource.variants.all(): z.writestr('%s/%s/%s' % (prefix, project.resources_path, variant.path), variant.get_contents()) manifest = generate_manifest(project, resources) manifest_name = manifest_name_for_project(project) z.writestr('%s/%s' % (prefix, manifest_name), manifest) if project.is_standard_project_type: # This file is always the same, but needed to build. z.writestr('%s/wscript' % prefix, generate_wscript_file(project, for_export=True)) z.writestr('%s/jshintrc' % prefix, generate_jshint_file(project))
def add_project_to_archive(z, project, prefix=""): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r"[^\w]+", "_", project.name).strip("_").lower() for source in source_files: src_dir = "worker_src" if source.target == "worker" else "src" z.writestr("%s/%s/%s" % (prefix, src_dir, source.file_name), source.get_contents()) for resource in resources: res_path = "resources" for variant in resource.variants.all(): z.writestr("%s/%s/%s" % (prefix, res_path, variant.path), variant.get_contents()) manifest = generate_manifest(project, resources) z.writestr("%s/appinfo.json" % prefix, manifest) if project.project_type == "native": # This file is always the same, but needed to build. z.writestr("%s/wscript" % prefix, generate_wscript_file(project, for_export=True)) z.writestr("%s/jshintrc" % prefix, generate_jshint_file(project))
def add_project_to_archive(z, project, prefix='', suffix=''): source_files = SourceFile.objects.filter(project=project) resources = ResourceFile.objects.filter(project=project) prefix += re.sub(r'[^\w]+', '_', project.name).strip('_').lower() prefix += suffix for source in source_files: path = os.path.join(prefix, source.project_path) z.writestr(path, source.get_contents()) for resource in resources: for variant in resource.variants.all(): z.writestr('%s/%s/%s' % (prefix, project.resources_path, variant.path), variant.get_contents()) manifest = generate_manifest(project, resources) manifest_name = manifest_name_for_project(project) z.writestr('%s/%s' % (prefix, manifest_name), manifest) if project.is_standard_project_type: # This file is always the same, but needed to build. z.writestr('%s/wscript' % prefix, generate_wscript_file(project, for_export=True)) z.writestr('%s/jshintrc' % prefix, generate_jshint_file(project))
def test_package_manifest(self): """ Check that the appinfo.json file generated from a project is functionally identical to a generated sample manifest. """ manifest = generate_manifest(self.project, []) self.check_appinfo_manifest(manifest)
def github_push(user, commit_message, repo_name, project): g = Github(user.github.token, client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET) repo = g.get_repo(repo_name) try: branch = repo.get_branch(project.github_branch or repo.master_branch) except GithubException: raise Exception("Unable to get branch.") commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) paths = [x.path for x in tree.tree] next_tree = {x.path: InputGitTreeElement(path=x.path, mode=x.mode, type=x.type, sha=x.sha) for x in tree.tree} try: root = find_project_root(paths) except: root = '' expected_paths = set() def update_expected_paths(new_path): # This adds the path *and* its parent directories to the list of expected paths. # The parent directories are already keys in next_tree, so if they aren't present in expected_paths # then, when iterating over next_tree to see which files have been deleted, we would have to treat # directories as special cases. split_path = new_path.split('/') expected_paths.update('/'.join(split_path[:p]) for p in range(2, len(split_path) + 1)) src_root = root + 'src/' worker_src_root = root + 'worker_src/' project_sources = project.source_files.all() has_changed = False for source in project_sources: repo_path = src_root + source.file_name if project.project_type == 'native': if source.target == 'worker': repo_path = worker_src_root + source.file_name elif project.app_modern_multi_js and source.file_name.endswith('.js'): repo_path = src_root + 'js/' + source.file_name update_expected_paths(repo_path) if repo_path not in next_tree: has_changed = True next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', content=source.get_contents()) print "New file: %s" % repo_path else: sha = next_tree[repo_path]._InputGitTreeElement__sha our_content = source.get_contents() expected_sha = git_sha(our_content) if expected_sha != sha: print "Updated file: %s" % repo_path next_tree[repo_path]._InputGitTreeElement__sha = NotSet next_tree[repo_path]._InputGitTreeElement__content = our_content has_changed = True # Now try handling resource files. resources = project.resources.all() resource_root = root + 'resources/' for res in resources: for variant in res.variants.all(): repo_path = resource_root + variant.path update_expected_paths(repo_path) if repo_path in next_tree: content = variant.get_contents() if git_sha(content) != next_tree[repo_path]._InputGitTreeElement__sha: print "Changed resource: %s" % repo_path has_changed = True blob = repo.create_git_blob(base64.b64encode(content), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path]._InputGitTreeElement__sha = blob.sha else: print "New resource: %s" % repo_path has_changed = True blob = repo.create_git_blob(base64.b64encode(variant.get_contents()), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', sha=blob.sha) # Manage deleted files for path in next_tree.keys(): if not (any(path.startswith(root) for root in (src_root, resource_root, worker_src_root))): continue if path not in expected_paths: del next_tree[path] print "Deleted file: %s" % path has_changed = True # Compare the resource dicts remote_manifest_path = root + 'appinfo.json' remote_wscript_path = root + 'wscript' remote_manifest_sha = next_tree[remote_manifest_path]._InputGitTreeElement__sha if remote_manifest_path in next_tree else None if remote_manifest_sha is not None: their_manifest_dict = json.loads(git_blob(repo, remote_manifest_sha)) their_res_dict = their_manifest_dict['resources'] else: their_manifest_dict = {} their_res_dict = {'media': []} our_manifest_dict = generate_manifest_dict(project, resources) our_res_dict = our_manifest_dict['resources'] if our_res_dict != their_res_dict: print "Resources mismatch." has_changed = True # Try removing things that we've deleted, if any to_remove = set(x['file'] for x in their_res_dict['media']) - set(x['file'] for x in our_res_dict['media']) for path in to_remove: repo_path = resource_root + path if repo_path in next_tree: print "Deleted resource: %s" % repo_path del next_tree[repo_path] # This one is separate because there's more than just the resource map changing. if their_manifest_dict != our_manifest_dict: has_changed = True if remote_manifest_path in next_tree: next_tree[remote_manifest_path]._InputGitTreeElement__sha = NotSet next_tree[remote_manifest_path]._InputGitTreeElement__content = generate_manifest(project, resources) else: next_tree[remote_manifest_path] = InputGitTreeElement(path=remote_manifest_path, mode='100644', type='blob', content=generate_manifest(project, resources)) if project.project_type == 'native' and remote_wscript_path not in next_tree: next_tree[remote_wscript_path] = InputGitTreeElement(path=remote_wscript_path, mode='100644', type='blob', content=generate_wscript_file(project, True)) has_changed = True # Commit the new tree. if has_changed: print "Has changed; committing" # GitHub seems to choke if we pass the raw directory nodes off to it, # so we delete those. for x in next_tree.keys(): if next_tree[x]._InputGitTreeElement__mode == '040000': del next_tree[x] print "removing subtree node %s" % x print [x._InputGitTreeElement__mode for x in next_tree.values()]
def github_push(user, commit_message, repo_name, project): g = Github(user.github.token, client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET) repo = g.get_repo(repo_name) try: branch = repo.get_branch(project.github_branch or repo.master_branch) except GithubException: raise Exception("Unable to get branch.") commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) paths = [x.path for x in tree.tree] next_tree = { x.path: InputGitTreeElement(path=x.path, mode=x.mode, type=x.type, sha=x.sha) for x in tree.tree } try: root = find_project_root(paths) except: root = '' src_root = root + 'src/' project_sources = project.source_files.all() has_changed = False for source in project_sources: repo_path = src_root + source.file_name if repo_path not in next_tree: has_changed = True next_tree[repo_path] = InputGitTreeElement( path=repo_path, mode='100644', type='blob', content=source.get_contents()) print "New file: %s" % repo_path else: sha = next_tree[repo_path]._InputGitTreeElement__sha our_content = source.get_contents() expected_sha = git_sha(our_content) if expected_sha != sha: print "Updated file: %s" % repo_path next_tree[repo_path]._InputGitTreeElement__sha = NotSet next_tree[ repo_path]._InputGitTreeElement__content = our_content has_changed = True expected_source_files = [src_root + x.file_name for x in project_sources] for path in next_tree.keys(): if not path.startswith(src_root): continue if path not in expected_source_files: del next_tree[path] print "Deleted file: %s" % path has_changed = True # Now try handling resource files. resources = project.resources.all() resource_root = root + 'resources/' for res in resources: repo_path = resource_root + res.path if repo_path in next_tree: content = res.get_contents() if git_sha( content) != next_tree[repo_path]._InputGitTreeElement__sha: print "Changed resource: %s" % repo_path has_changed = True blob = repo.create_git_blob(base64.b64encode(content), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path]._InputGitTreeElement__sha = blob.sha else: print "New resource: %s" % repo_path blob = repo.create_git_blob(base64.b64encode(res.get_contents()), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', sha=blob.sha) remote_manifest_path = root + 'appinfo.json' remote_wscript_path = root + 'wscript' remote_manifest_sha = next_tree[ remote_manifest_path]._InputGitTreeElement__sha if remote_manifest_path in next_tree else None if remote_manifest_sha is not None: their_manifest_dict = json.loads(git_blob(repo, remote_manifest_sha)) their_res_dict = their_manifest_dict['resources'] else: their_manifest_dict = {} their_res_dict = {'media': []} our_manifest_dict = generate_manifest_dict(project, resources) our_res_dict = our_manifest_dict['resources'] if our_res_dict != their_res_dict: print "Resources mismatch." has_changed = True # Try removing things that we've deleted, if any to_remove = set(x['file'] for x in their_res_dict['media']) - set( x['file'] for x in our_res_dict['media']) for path in to_remove: repo_path = resource_root + path if repo_path in next_tree: print "Deleted resource: %s" % repo_path del next_tree[repo_path] # This one is separate because there's more than just the resource map changing. if their_manifest_dict != our_manifest_dict: if remote_manifest_path in next_tree: next_tree[remote_manifest_path]._InputGitTreeElement__sha = NotSet next_tree[ remote_manifest_path]._InputGitTreeElement__content = generate_manifest( project, resources) else: next_tree[remote_manifest_path] = InputGitTreeElement( path=remote_manifest_path, mode='100644', type='blob', content=generate_manifest(project, resources)) if project.project_type == 'native' and remote_wscript_path not in next_tree: next_tree[remote_wscript_path] = InputGitTreeElement( path=remote_wscript_path, mode='100644', type='blob', content=generate_wscript_file(project, True)) has_changed = True # Commit the new tree. if has_changed: print "Has changed; committing" # GitHub seems to choke if we pass the raw directory nodes off to it, # so we delete those. for x in next_tree.keys(): if next_tree[x]._InputGitTreeElement__mode == '040000': del next_tree[x] print "removing subtree node %s" % x print[x._InputGitTreeElement__mode for x in next_tree.values()] git_tree = repo.create_git_tree(next_tree.values()) print "Created tree %s" % git_tree.sha git_commit = repo.create_git_commit(commit_message, git_tree, [commit]) print "Created commit %s" % git_commit.sha git_ref = repo.get_git_ref( 'heads/%s' % (project.github_branch or repo.master_branch)) git_ref.edit(git_commit.sha) print "Updated ref %s" % git_ref.ref project.github_last_commit = git_commit.sha project.github_last_sync = now() project.save() return True send_keen_event('cloudpebble', 'cloudpebble_github_push', user=user, data={'data': { 'repo': project.github_repo }}) return False
def github_push(user, commit_message, repo_name, project): g = Github(user.github.token, client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET) repo = g.get_repo(repo_name) try: branch = repo.get_branch(project.github_branch or repo.master_branch) except GithubException: raise Exception("Unable to get branch.") commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) next_tree = {x.path: InputGitTreeElement(path=x.path, mode=x.mode, type=x.type, sha=x.sha) for x in tree.tree} try: root, manifest_item = find_project_root_and_manifest([GitProjectItem(repo, x) for x in tree.tree]) except InvalidProjectArchiveException: root = '' manifest_item = None expected_paths = set() def update_expected_paths(new_path): # This adds the path *and* its parent directories to the list of expected paths. # The parent directories are already keys in next_tree, so if they aren't present in expected_paths # then, when iterating over next_tree to see which files have been deleted, we would have to treat # directories as special cases. split_path = new_path.split('/') expected_paths.update('/'.join(split_path[:p]) for p in range(2, len(split_path) + 1)) src_root = root + 'src/' worker_src_root = root + 'worker_src/' project_sources = project.source_files.all() has_changed = False for source in project_sources: repo_path = src_root + source.file_name if project.project_type == 'native': if source.target == 'worker': repo_path = worker_src_root + source.file_name elif project.app_modern_multi_js and source.file_name.endswith('.js'): repo_path = src_root + 'js/' + source.file_name update_expected_paths(repo_path) if repo_path not in next_tree: has_changed = True next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', content=source.get_contents()) logger.debug("New file: %s", repo_path) else: sha = next_tree[repo_path]._InputGitTreeElement__sha our_content = source.get_contents() expected_sha = git_sha(our_content) if expected_sha != sha: logger.debug("Updated file: %s", repo_path) next_tree[repo_path]._InputGitTreeElement__sha = NotSet next_tree[repo_path]._InputGitTreeElement__content = our_content has_changed = True # Now try handling resource files. resources = project.resources.all() resource_root = root + 'resources/' for res in resources: for variant in res.variants.all(): repo_path = resource_root + variant.path update_expected_paths(repo_path) if repo_path in next_tree: content = variant.get_contents() if git_sha(content) != next_tree[repo_path]._InputGitTreeElement__sha: logger.debug("Changed resource: %s", repo_path) has_changed = True blob = repo.create_git_blob(base64.b64encode(content), 'base64') logger.debug("Created blob %s", blob.sha) next_tree[repo_path]._InputGitTreeElement__sha = blob.sha else: logger.debug("New resource: %s", repo_path) has_changed = True blob = repo.create_git_blob(base64.b64encode(variant.get_contents()), 'base64') logger.debug("Created blob %s", blob.sha) next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', sha=blob.sha) # Manage deleted files for path in next_tree.keys(): if not (any(path.startswith(root) for root in (src_root, resource_root, worker_src_root))): continue if path not in expected_paths: del next_tree[path] logger.debug("Deleted file: %s", path) has_changed = True # Compare the resource dicts remote_manifest_path = root + manifest_name_for_project(project) remote_wscript_path = root + 'wscript' if manifest_item: their_manifest_dict = json.loads(manifest_item.read()) their_res_dict = their_manifest_dict.get('resources', their_manifest_dict.get('pebble', their_manifest_dict).get('resources', {'media': []})) # If the manifest needs a new path (e.g. it is now package.json), delete the old one if manifest_item.path != remote_manifest_path: del next_tree[manifest_item.path] else: their_manifest_dict = {} their_res_dict = {'media': []} our_manifest_dict = generate_manifest_dict(project, resources) our_res_dict = our_manifest_dict.get('resources', our_manifest_dict.get('pebble', our_manifest_dict).get('resources', {'media': []})) if our_res_dict != their_res_dict: logger.debug("Resources mismatch.") has_changed = True # Try removing things that we've deleted, if any to_remove = set(x['file'] for x in their_res_dict['media']) - set(x['file'] for x in our_res_dict['media']) for path in to_remove: repo_path = resource_root + path if repo_path in next_tree: logger.debug("Deleted resource: %s", repo_path) del next_tree[repo_path] # This one is separate because there's more than just the resource map changing. if their_manifest_dict != our_manifest_dict: has_changed = True if remote_manifest_path in next_tree: next_tree[remote_manifest_path]._InputGitTreeElement__sha = NotSet next_tree[remote_manifest_path]._InputGitTreeElement__content = generate_manifest(project, resources) else: next_tree[remote_manifest_path] = InputGitTreeElement(path=remote_manifest_path, mode='100644', type='blob', content=generate_manifest(project, resources)) if project.project_type == 'native' and remote_wscript_path not in next_tree: next_tree[remote_wscript_path] = InputGitTreeElement(path=remote_wscript_path, mode='100644', type='blob', content=generate_wscript_file(project, True)) has_changed = True # Commit the new tree. if has_changed: logger.debug("Has changed; committing") # GitHub seems to choke if we pass the raw directory nodes off to it, # so we delete those. for x in next_tree.keys(): if next_tree[x]._InputGitTreeElement__mode == '040000': del next_tree[x] logger.debug("removing subtree node %s", x) logger.debug([x._InputGitTreeElement__mode for x in next_tree.values()]) git_tree = repo.create_git_tree(next_tree.values()) logger.debug("Created tree %s", git_tree.sha) git_commit = repo.create_git_commit(commit_message, git_tree, [commit]) logger.debug("Created commit %s", git_commit.sha) git_ref = repo.get_git_ref('heads/%s' % (project.github_branch or repo.master_branch)) git_ref.edit(git_commit.sha) logger.debug("Updated ref %s", git_ref.ref) project.github_last_commit = git_commit.sha project.github_last_sync = now() project.save() return True send_td_event('cloudpebble_github_push', data={ 'data': { 'repo': project.github_repo } }, user=user) return False
def github_push(user, commit_message, repo_name, project): g = Github(user.github.token, client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET) repo = g.get_repo(repo_name) try: branch = repo.get_branch(project.github_branch or repo.master_branch) except GithubException: raise Exception("Unable to get branch.") commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) next_tree = {x.path: InputGitTreeElement(path=x.path, mode=x.mode, type=x.type, sha=x.sha) for x in tree.tree} try: root, manifest_item = find_project_root_and_manifest([GitProjectItem(repo, x) for x in tree.tree]) except InvalidProjectArchiveException: root = '' manifest_item = None expected_paths = set() def update_expected_paths(new_path): # This adds the path *and* its parent directories to the list of expected paths. # The parent directories are already keys in next_tree, so if they aren't present in expected_paths # then, when iterating over next_tree to see which files have been deleted, we would have to treat # directories as special cases. split_path = new_path.split('/') expected_paths.update('/'.join(split_path[:p]) for p in range(2, len(split_path) + 1)) project_sources = project.source_files.all() has_changed = False for source in project_sources: repo_path = os.path.join(root, source.project_path) update_expected_paths(repo_path) if repo_path not in next_tree: has_changed = True next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', content=source.get_contents()) logger.debug("New file: %s", repo_path) else: sha = next_tree[repo_path]._InputGitTreeElement__sha our_content = source.get_contents() expected_sha = git_sha(our_content) if expected_sha != sha: logger.debug("Updated file: %s", repo_path) next_tree[repo_path]._InputGitTreeElement__sha = NotSet next_tree[repo_path]._InputGitTreeElement__content = our_content has_changed = True # Now try handling resource files. resources = project.resources.all() resource_root = project.resources_path for res in resources: for variant in res.variants.all(): repo_path = os.path.join(resource_root, variant.path) update_expected_paths(repo_path) if repo_path in next_tree: content = variant.get_contents() if git_sha(content) != next_tree[repo_path]._InputGitTreeElement__sha: logger.debug("Changed resource: %s", repo_path) has_changed = True blob = repo.create_git_blob(base64.b64encode(content), 'base64') logger.debug("Created blob %s", blob.sha) next_tree[repo_path]._InputGitTreeElement__sha = blob.sha else: logger.debug("New resource: %s", repo_path) has_changed = True blob = repo.create_git_blob(base64.b64encode(variant.get_contents()), 'base64') logger.debug("Created blob %s", blob.sha) next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', sha=blob.sha) # Manage deleted files src_root = os.path.join(root, 'src') worker_src_root = os.path.join(root, 'worker_src') for path in next_tree.keys(): if not (any(path.startswith(root+'/') for root in (src_root, resource_root, worker_src_root))): continue if path not in expected_paths: del next_tree[path] logger.debug("Deleted file: %s", path) has_changed = True # Compare the resource dicts remote_manifest_path = root + manifest_name_for_project(project) remote_wscript_path = root + 'wscript' if manifest_item: their_manifest_dict = json.loads(manifest_item.read()) their_res_dict = their_manifest_dict.get('resources', their_manifest_dict.get('pebble', their_manifest_dict).get('resources', {'media': []})) # If the manifest needs a new path (e.g. it is now package.json), delete the old one if manifest_item.path != remote_manifest_path: del next_tree[manifest_item.path] else: their_manifest_dict = {} their_res_dict = {'media': []} our_manifest_dict = generate_manifest_dict(project, resources) our_res_dict = our_manifest_dict.get('resources', our_manifest_dict.get('pebble', our_manifest_dict).get('resources', {'media': []})) if our_res_dict != their_res_dict: logger.debug("Resources mismatch.") has_changed = True # Try removing things that we've deleted, if any to_remove = set(x['file'] for x in their_res_dict['media']) - set(x['file'] for x in our_res_dict['media']) for path in to_remove: repo_path = resource_root + path if repo_path in next_tree: logger.debug("Deleted resource: %s", repo_path) del next_tree[repo_path] # This one is separate because there's more than just the resource map changing. if their_manifest_dict != our_manifest_dict: has_changed = True if remote_manifest_path in next_tree: next_tree[remote_manifest_path]._InputGitTreeElement__sha = NotSet next_tree[remote_manifest_path]._InputGitTreeElement__content = generate_manifest(project, resources) else: next_tree[remote_manifest_path] = InputGitTreeElement(path=remote_manifest_path, mode='100644', type='blob', content=generate_manifest(project, resources)) if project.project_type == 'native' and remote_wscript_path not in next_tree: next_tree[remote_wscript_path] = InputGitTreeElement(path=remote_wscript_path, mode='100644', type='blob', content=generate_wscript_file(project, True)) has_changed = True # Commit the new tree. if has_changed: logger.debug("Has changed; committing") # GitHub seems to choke if we pass the raw directory nodes off to it, # so we delete those. for x in next_tree.keys(): if next_tree[x]._InputGitTreeElement__mode == '040000': del next_tree[x] logger.debug("removing subtree node %s", x) logger.debug([x._InputGitTreeElement__mode for x in next_tree.values()]) git_tree = repo.create_git_tree(next_tree.values()) logger.debug("Created tree %s", git_tree.sha) git_commit = repo.create_git_commit(commit_message, git_tree, [commit]) logger.debug("Created commit %s", git_commit.sha) git_ref = repo.get_git_ref('heads/%s' % (project.github_branch or repo.master_branch)) git_ref.edit(git_commit.sha) logger.debug("Updated ref %s", git_ref.ref) project.github_last_commit = git_commit.sha project.github_last_sync = now() project.save() return True send_td_event('cloudpebble_github_push', data={ 'data': { 'repo': project.github_repo } }, user=user) return False
def test_package_manifest(self): """ Check that the manifest create for a project is functionally identical to a generated sample manifest. """ manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest)
def test_package_manifest_with_dependencies(self): """ Check that dependencies are represented in the package.json file. """ deps = {"some_package": "1.2.3", "another": "^4.2.0"} self.project.set_dependencies(deps) manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={"dependencies": deps})
def test_manifest_short_name(self): """ Check that app_short_name is mangled into a valid npm package name """ self.project.app_short_name = "_CAPITALS_and...dots-and - spaces ! " manifest = generate_manifest(self.project, []) self.check_package_manifest(manifest, package_options={"name": "capitals_and...dots-and-spaces"})
def github_push(user, commit_message, repo_name, project): g = Github(user.github.token, client_id=settings.GITHUB_CLIENT_ID, client_secret=settings.GITHUB_CLIENT_SECRET) repo = g.get_repo(repo_name) try: branch = repo.get_branch(project.github_branch or repo.master_branch) except GithubException: raise Exception("Unable to get branch.") commit = repo.get_git_commit(branch.commit.sha) tree = repo.get_git_tree(commit.tree.sha, recursive=True) paths = [x.path for x in tree.tree] next_tree = {x.path: InputGitTreeElement(path=x.path, mode=x.mode, type=x.type, sha=x.sha) for x in tree.tree} try: root = find_project_root(paths) except: root = '' src_root = root + 'src/' project_sources = project.source_files.all() has_changed = False for source in project_sources: repo_path = src_root + source.file_name if repo_path not in next_tree: has_changed = True next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', content=source.get_contents()) print "New file: %s" % repo_path else: sha = next_tree[repo_path]._InputGitTreeElement__sha our_content = source.get_contents() expected_sha = git_sha(our_content) if expected_sha != sha: print "Updated file: %s" % repo_path next_tree[repo_path]._InputGitTreeElement__sha = NotSet next_tree[repo_path]._InputGitTreeElement__content = our_content has_changed = True expected_source_files = [src_root + x.file_name for x in project_sources] for path in next_tree.keys(): if not path.startswith(src_root): continue if path not in expected_source_files: del next_tree[path] print "Deleted file: %s" % path has_changed = True # Now try handling resource files. resources = project.resources.all() resource_root = root + 'resources/' for res in resources: for variant in res.variants.all(): repo_path = resource_root + variant.path if repo_path in next_tree: content = variant.get_contents() if git_sha(content) != next_tree[repo_path]._InputGitTreeElement__sha: print "Changed resource: %s" % repo_path has_changed = True blob = repo.create_git_blob(base64.b64encode(content), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path]._InputGitTreeElement__sha = blob.sha else: print "New resource: %s" % repo_path blob = repo.create_git_blob(base64.b64encode(variant.get_contents()), 'base64') print "Created blob %s" % blob.sha next_tree[repo_path] = InputGitTreeElement(path=repo_path, mode='100644', type='blob', sha=blob.sha) remote_manifest_path = root + 'appinfo.json' remote_wscript_path = root + 'wscript' remote_manifest_sha = next_tree[remote_manifest_path]._InputGitTreeElement__sha if remote_manifest_path in next_tree else None if remote_manifest_sha is not None: their_manifest_dict = json.loads(git_blob(repo, remote_manifest_sha)) their_res_dict = their_manifest_dict['resources'] else: their_manifest_dict = {} their_res_dict = {'media': []} our_manifest_dict = generate_manifest_dict(project, resources) our_res_dict = our_manifest_dict['resources'] if our_res_dict != their_res_dict: print "Resources mismatch." has_changed = True # Try removing things that we've deleted, if any to_remove = set(x['file'] for x in their_res_dict['media']) - set(x['file'] for x in our_res_dict['media']) for path in to_remove: repo_path = resource_root + path if repo_path in next_tree: print "Deleted resource: %s" % repo_path del next_tree[repo_path] # This one is separate because there's more than just the resource map changing. if their_manifest_dict != our_manifest_dict: if remote_manifest_path in next_tree: next_tree[remote_manifest_path]._InputGitTreeElement__sha = NotSet next_tree[remote_manifest_path]._InputGitTreeElement__content = generate_manifest(project, resources) else: next_tree[remote_manifest_path] = InputGitTreeElement(path=remote_manifest_path, mode='100644', type='blob', content=generate_manifest(project, resources)) if project.project_type == 'native' and remote_wscript_path not in next_tree: next_tree[remote_wscript_path] = InputGitTreeElement(path=remote_wscript_path, mode='100644', type='blob', content=generate_wscript_file(project, True)) has_changed = True # Commit the new tree. if has_changed: print "Has changed; committing" # GitHub seems to choke if we pass the raw directory nodes off to it, # so we delete those. for x in next_tree.keys(): if next_tree[x]._InputGitTreeElement__mode == '040000': del next_tree[x] print "removing subtree node %s" % x print [x._InputGitTreeElement__mode for x in next_tree.values()]