def test_can_request_project_instructions_for_id(self): for (project_id, expected_project_instructions ) in TEST_PROJECT_ID_TO_INSTRUCTIONS_MAP.iteritems(): extracted_project_instructions = scratchwebapi.getMetaDataEntry( project_id, 'instructions') assert extracted_project_instructions== expected_project_instructions, \ "'{}' is not equal to '{}'".format(extracted_project_instructions, expected_project_instructions)
def test_can_request_project_title_for_id(self): for (project_id, expected_project_title ) in TEST_PROJECT_ID_TO_TITLE_MAP.iteritems(): extracted_project_title = scratchwebapi.getMetaDataEntry( project_id, 'title') assert extracted_project_title is not None assert extracted_project_title == expected_project_title, \ "'{}' is not equal to '{}'".format(extracted_project_title, expected_project_title)
def test_can_request_project_notes_and_credits_for_id(self): for (project_id, expected_project_notes_and_credits ) in TEST_PROJECT_ID_TO_NOTES_AND_CREDITS_MAP.iteritems(): extracted_project_notes_and_credits = scratchwebapi.getMetaDataEntry( project_id, 'description') assert extracted_project_notes_and_credits is not None assert extracted_project_notes_and_credits == expected_project_notes_and_credits, \ "'{}' is not equal to '{}'".format(extracted_project_notes_and_credits, expected_project_notes_and_credits)
def test_can_request_project_owner_for_id(self): for (project_id, expected_project_owner ) in TEST_PROJECT_ID_TO_OWNER_MAP.iteritems(): extracted_project_owner = scratchwebapi.getMetaDataEntry( project_id, 'username') assert extracted_project_owner is not None assert extracted_project_owner == expected_project_owner, \ "'{}' is not equal to '{}'".format(extracted_project_owner, expected_project_owner)
def assertMainSuccess(self, args, project_id): output_path = self._testresult_folder_path if len(args) == 1: args += [output_path] return_val = self.execute_run_script(args) assert return_val == helpers.ExitCode.SUCCESS project_name = scratchwebapi.getMetaDataEntry(project_id, "title") self.assertValidCatrobatProgramPackageAndUnpackIf(converter.ConvertedProject._converted_output_path(output_path, project_name), project_name)
def test_can_detect_correct_visibility_state_of_project(self): project_visibility_map = { "107178598": scratchwebapi.ScratchProjectVisibiltyState.PRIVATE, "123242912": scratchwebapi.ScratchProjectVisibiltyState.PRIVATE, "95106124": scratchwebapi.ScratchProjectVisibiltyState.PUBLIC, "85594786": scratchwebapi.ScratchProjectVisibiltyState.PUBLIC } for (project_id, expected_visibility_state) in project_visibility_map.iteritems(): detected_visibility_state = scratchwebapi.getMetaDataEntry( project_id, 'visibility') assert expected_visibility_state == detected_visibility_state
def convert_scratch_project(job_ID, host, port, verbose): logging.basicConfig( filename=None, level=logging.DEBUG, format='%(asctime)s: %(levelname)7s: [%(name)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') # job = get_current_job() # job.meta['handled_by'] = socket.gethostname() # job.save() # validate URL if job_ID == None or not isinstance(job_ID, int): _logger.error( "No or invalid Scratch project ID given: {}".format(job_ID)) return if not os.path.isfile(CERTIFICATE_PATH): _logger.error("Cannot find server certificate: %s", CERTIFICATE_PATH) return # retries = int(helpers.config.get("SCRATCH_API", "http_retries")) # timeout_in_secs = int(helpers.config.get("SCRATCH_API", "http_timeout")) / 1000 # backoff = int(helpers.config.get("SCRATCH_API", "http_backoff")) # delay = int(helpers.config.get("SCRATCH_API", "http_delay")) # user_agent = helpers.config.get("SCRATCH_API", "user_agent") # # # preprocessing: fetch project title and project image URL via web API # def retry_hook(exc, tries, delay): # _logger.warning(" Exception: {}\nRetrying after {}:'{}' in {} secs (remaining trys: {})" \ # .format(sys.exc_info()[0], type(exc).__name__, exc, delay, tries)) # # @helpers.retry((urllib2.URLError, socket.timeout, IOError, BadStatusLine), delay=delay, # backoff=backoff, tries=retries, hook=retry_hook) # def read_content_of_url(url): # _logger.info("Fetching project title from: {}".format(scratch_project_url)) # req = urllib2.Request(url, headers={ "User-Agent": user_agent }) # return urllib2.urlopen(req, timeout=timeout_in_secs).read() title = None image_URL = None scratch_project_url = "%s%d" % (SCRATCH_PROJECT_META_DATA_BASE_URL, job_ID) try: # html_content = read_content_of_url(scratch_project_url) # document = webhelpers.ResponseBeautifulSoupDocumentWrapper(BeautifulSoup(html_content.decode('utf-8', 'ignore'), b'html5lib')) # document = document.wrapped_document.text() title, image_URL = scratchwebapi.getMetaDataEntry( job_ID, "title", "image") # image_URL = scratchwebapi.getMetaDataEntry(job_ID, "image") if title == None: raise Warning("Unable to set title of project from the project's website!" \ " Reason: Cannot parse title from returned html content!") if image_URL == None: raise Warning("Unable to extract image url of project from the project's website!" \ " Reason: Cannot parse image url from returned html content!") except: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] # log error and continue without updating title and/or image URL! _logger.error("Unexpected error for URL: {}, {}, {}, {}, {}".format(scratch_project_url, \ sys.exc_info()[0], exc_type, fname, str(exc_tb.tb_lineno))) _logger.info("Project title is: {}".format(title)) args = { "url": scratch_project_url, "jobID": job_ID, "title": title, "imageURL": image_URL, "outputDir": helpers.config.get("PATHS", "web_output") } # set up signal handler #signal.signal(signal.SIGTERM, sig_handler) #signal.signal(signal.SIGINT, sig_handler) ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) #@UndefinedVariable ssl_ctx.verify_mode = ssl.CERT_REQUIRED # check only hostnames for non-local servers ssl_ctx.check_hostname = (host != "localhost") ssl_ctx.load_verify_locations(cafile=CERTIFICATE_PATH) handler = ConverterJobHandler(host, port, verbose, AUTH_KEY, ssl_options=ssl_ctx) handler.run(args) IOLoop.instance().start()
def __init__(self, project_base_path, name=None, project_id=None, progress_bar=None, is_local_project=False): def read_md5_to_resource_path_mapping(): md5_to_resource_path_map = {} # TODO: clarify that only files with extension are covered for res_file_path in glob.glob( os.path.join(project_base_path, "*.*")): resource_name = common.md5_hash( res_file_path) + os.path.splitext(res_file_path)[1] md5_to_resource_path_map[resource_name] = res_file_path try: # penLayer is no regular resource file del md5_to_resource_path_map[self['penLayerMD5']] except KeyError: # TODO: include penLayer download in webapi pass assert self['penLayerMD5'] not in md5_to_resource_path_map return md5_to_resource_path_map super(Project, self).__init__( self.raw_project_code_from_project_folder_path(project_base_path)) self.project_base_path = project_base_path self.project_id = self.get_info().get( "projectID") if project_id is None else project_id if not is_local_project: self.downloadScratch2ProjectResources(project_base_path, progress_bar) if not self.project_id: self.project_id = "0" self.name = name if name is not None else "Untitled" self.instructions = self.notes_and_credits = None self.automatic_screenshot_image_url = None else: if name is not None: self.name = name else: [self.name ] = scratchwebapi.getMetaDataEntry(self.project_id, "title") # self.name = name if name is not None else scratchwebapi.getMetaDataEntry(self.project_id, "title") self.instructions, self.notes_and_credits, self.automatic_screenshot_image_url =\ scratchwebapi.getMetaDataEntry(self.project_id, "instructions", "description", "image") # self.instructions = scratchwebapi.getMetaDataEntry(self.project_id, "instructions") # self.notes_and_credits = scratchwebapi.getMetaDataEntry(self.project_id, "description") # self.automatic_screenshot_image_url = "{}{}.png".format(scratchwebapi.SCRATCH_PROJECT_IMAGE_BASE_URL, self.project_id) if progress_bar != None: progress_bar.update(ProgressType.DETAILS) # details step passed _log.info( "Scratch project: %s%s", self.name, "(ID: {})".format(self.project_id) if self.project_id > 0 else "") self.name = self.name.strip( ) if self.name != None else "Unknown Project" self.md5_to_resource_path_map = read_md5_to_resource_path_mapping() self.global_user_lists = self.objects[0].get_lists() for scratch_object in self.objects: verify_resources_of_scratch_object(scratch_object, self.md5_to_resource_path_map, self.project_base_path) listened_keys = [] for scratch_obj in self.objects: for script in scratch_obj.scripts: if script.type == SCRIPT_KEY_PRESSED: assert len(script.arguments) == 1 listened_keys += [(argument, "listenedKeys") for argument in script.arguments] try: self.listened_keys.update(listened_keys) except AttributeError: self.listened_keys = set(listened_keys) # TODO: rename self.background_md5_names = set( [costume[JsonKeys.COSTUME_MD5] for costume in self.get_costumes()]) result = self.find_unused_resources_name_and_filepath() self.unused_resource_names = result[0] if len(result) > 0 else [] self.unused_resource_paths = result[1] if len(result) > 0 else [] for unused_path in self.unused_resource_paths: _log.warning("Project folder contains unused resource file: '%s'. These " \ "will be omitted for Catrobat project.", os.path.basename(unused_path))
def run_converter(scratch_project_file_or_url, output_dir, extract_resulting_catrobat=False, temp_rm=True, show_version_only=False, show_info_only=False, archive_name=None, web_mode=False): def check_base_environment(): if "java" not in sys.platform: raise EnvironmentError("Must be called with Jython interpreter.") if System.getProperty(helpers.JYTHON_RESPECT_JAVA_ACCESSIBILITY_PROPERTY) != 'false': raise EnvironmentError("Jython registry property '%s' must be set to 'false'." % helpers.JYTHON_RESPECT_JAVA_ACCESSIBILITY_PROPERTY) def check_converter_environment(): # TODO: refactor to combined class with explicit environment check method tools.svgtopng._checked_batik_jar_path() tools.wavconverter._checked_sox_path() try: from java.io import IOError from java.lang import System except ImportError: log.error("Must be called with Jython interpreter.") return helpers.ExitCode.FAILURE # nested import to be able to check for Jython interpreter first from scratchtocatrobat import tools from scratchtocatrobat.tools import common from scratchtocatrobat.converter import converter, catrobat from scratchtocatrobat.scratch import scratchwebapi, scratch try: check_base_environment() check_converter_environment() catrobat_language_version_from_config = float(helpers.catrobat_info("catrobat_language_version")) if catrobat_language_version_from_config != catrobat.CATROBAT_LANGUAGE_VERSION: raise RuntimeError("Wrong Catrobat Language version set in config-file! " \ "Catrobat language version is %.3f but should be %.3f! Please update!" % (catrobat_language_version_from_config, catrobat.CATROBAT_LANGUAGE_VERSION)) tag_name = helpers.tag_name_of_used_catroid_hierarchy() latest_release_data = helpers.latest_catroid_repository_release_data() if show_version_only or show_info_only: helpers.print_info_or_version_screen(show_version_only, catrobat.CATROBAT_LANGUAGE_VERSION) return helpers.ExitCode.SUCCESS elif latest_release_data and not web_mode: # suppress release-reminder in web-mode current_release_version = helpers.extract_version_number(tag_name) latest_release_version = helpers.extract_version_number(latest_release_data["tag_name"]) if current_release_version < latest_release_version: print("Latest Catroid release: %s (%s)" % (latest_release_data["tag_name"], latest_release_data["published_at"])) print("%sA NEW CATROID RELEASE IS AVAILABLE!\nPLEASE UPDATE THE CLASS HIERARCHY " \ "OF THE CONVERTER FROM CATROID VERSION %s TO VERSION %s%s" % ( helpers.cli_colors.FAIL, tag_name, latest_release_data["tag_name"], helpers.cli_colors.ENDC )) log.info("calling converter") if not os.path.isdir(output_dir): raise EnvironmentError("Output folder must be a directory, but is %s" % output_dir) scratch3ProjectName = "Untitled" progress_bar = helpers.ProgressBar(None, web_mode, sys.stdout) with common.TemporaryDirectory(remove_on_exit=temp_rm) as scratch_project_dir: is_local_project = True project_id = None if scratch_project_file_or_url.startswith("https://"): is_local_project = False validate_scratch_url(scratch_project_file_or_url) project_id = scratchwebapi.extract_project_id_from_url(scratch_project_file_or_url) if not scratchwebapi.request_is_project_available(project_id): raise common.ScratchtobatError("Project with ID %s not available" % project_id) visibility = scratchwebapi.getMetaDataEntry(project_id, "visibility") if visibility != scratchwebapi.ScratchProjectVisibiltyState.PUBLIC: log.warn('-'*80) log.warn("CAVE: Project with ID %s is NOT a public project!! Trying to " \ "continue the conversion-process anyway, but expecting the " \ "conversion to fail or to be incomplete...", project_id) log.warn('-'*80) log.info("Downloading project from URL: '{}' to temp dir {} ...".format( scratch_project_file_or_url, scratch_project_dir)) scratchwebapi.download_project(scratch_project_file_or_url, scratch_project_dir, progress_bar) scratch3ProjectName = scratchwebapi.getMetaDataEntry(project_id, "title") elif os.path.isfile(scratch_project_file_or_url): log.info("Extracting project from path: '{}' ...".format(scratch_project_file_or_url)) common.extract(scratch_project_file_or_url, scratch_project_dir) else: if not os.path.isdir(scratch_project_file_or_url): raise common.ScratchtobatError("Directory of local project not found in %s" % scratch_project_file_or_url) log.info("Loading project from path: '{}' ...".format(scratch_project_file_or_url)) scratch_project_dir = scratch_project_file_or_url isScratch3Project = scratch3.is_scratch3_project(scratch_project_dir) if isScratch3Project: scratch3.convert_to_scratch2_data(scratch_project_dir, project_id) project = scratch.Project(scratch_project_dir, progress_bar=progress_bar, is_local_project = is_local_project) if isScratch3Project: project.name = scratch3ProjectName log.info("Converting scratch project '%s' into output folder: %s", project.name, output_dir) context = converter.Context() converted_project = converter.converted(project, progress_bar, context) catrobat_program_path = converted_project.save_as_catrobat_package_to(output_dir, archive_name, progress_bar, context) if extract_resulting_catrobat: extraction_path = os.path.join(output_dir, os.path.splitext(os.path.basename(catrobat_program_path))[0]) common.rm_dir(extraction_path) common.makedirs(extraction_path) scratch_output_path = os.path.join(extraction_path, "scratch") common.copy_dir(scratch_project_dir, scratch_output_path, overwrite=True) common.extract(catrobat_program_path, extraction_path) progress_bar.finish() except (common.ScratchtobatError, EnvironmentError, IOError) as e: log.error(e) return helpers.ExitCode.FAILURE except Exception as e: log.exception(e) return helpers.ExitCode.FAILURE return helpers.ExitCode.SUCCESS
def test_can_request_project_title_and_image_for_id(self): extracted_project_title, image = scratchwebapi.getMetaDataEntry( 10205819, "title", "image") assert extracted_project_title == "Dancin' in the Castle" assert image == "https://cdn2.scratch.mit.edu/get_image/project/10205819_480x360.png"
def convert_scratch_project(job_ID, host, port, verbose): logging.basicConfig( filename=None, level=logging.DEBUG, format='%(asctime)s: %(levelname)7s: [%(name)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') # job = get_current_job() # job.meta['handled_by'] = socket.gethostname() # job.save() # validate URL if job_ID == None or not isinstance(job_ID, int): _logger.error( "No or invalid Scratch project ID given: {}".format(job_ID)) return if not os.path.isfile(CERTIFICATE_PATH): _logger.error("Cannot find server certificate: %s", CERTIFICATE_PATH) return title = None image_URL = None scratch_project_url = "%s%d" % (SCRATCH_PROJECT_META_DATA_BASE_URL, job_ID) try: title, image_URL = scratchwebapi.getMetaDataEntry( job_ID, "title", "image") if title == None: raise Warning("Unable to set title of project from the project's website!" \ " Reason: Cannot parse title from returned html content!") if image_URL == None: raise Warning("Unable to extract image url of project from the project's website!" \ " Reason: Cannot parse image url from returned html content!") except: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] # log error and continue without updating title and/or image URL! _logger.error("Unexpected error for URL: {}, {}, {}, {}, {}".format(scratch_project_url, \ sys.exc_info()[0], exc_type, fname, str(exc_tb.tb_lineno))) _logger.info("Project title is: {}".format(title)) args = { "url": scratch_project_url, "jobID": job_ID, "title": title, "imageURL": image_URL, "outputDir": helpers.config.get("PATHS", "web_output") } # set up signal handler #signal.signal(signal.SIGTERM, sig_handler) #signal.signal(signal.SIGINT, sig_handler) ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) #@UndefinedVariable ssl_ctx.verify_mode = ssl.CERT_REQUIRED # check only hostnames for non-local servers ssl_ctx.check_hostname = (host != "localhost") ssl_ctx.load_verify_locations(cafile=CERTIFICATE_PATH) handler = ConverterJobHandler(host, port, verbose, AUTH_KEY, ssl_options=ssl_ctx) handler.run(args) IOLoop.instance().start()
def __init__(self, project_base_path, name=None, project_id=None, progress_bar=None, is_local_project=False): def read_md5_to_resource_path_mapping(): md5_to_resource_path_map = {} # TODO: clarify that only files with extension are covered for res_file_path in glob.glob(os.path.join(project_base_path, "*.*")): resource_name = common.md5_hash(res_file_path) + os.path.splitext(res_file_path)[1] md5_to_resource_path_map[resource_name] = res_file_path try: # penLayer is no regular resource file del md5_to_resource_path_map[self['penLayerMD5']] except KeyError: # TODO: include penLayer download in webapi pass assert self['penLayerMD5'] not in md5_to_resource_path_map return md5_to_resource_path_map super(Project, self).__init__(self.raw_project_code_from_project_folder_path(project_base_path)) self.project_base_path = project_base_path self.project_id = self.get_info().get("projectID") if project_id is None else project_id if not is_local_project: self.downloadScratch2ProjectResources(project_base_path, progress_bar) if not self.project_id: self.project_id = "0" self.name = name if name is not None else "Untitled" self.instructions = self.notes_and_credits = None self.automatic_screenshot_image_url = None else: if name is not None: self.name = name else: self.name = scratchwebapi.getMetaDataEntry(self.project_id, "title") # self.name = name if name is not None else scratchwebapi.getMetaDataEntry(self.project_id, "title") self.instructions, self.notes_and_credits, self.automatic_screenshot_image_url =\ scratchwebapi.getMetaDataEntry(self.project_id, "instructions", "description", "image") # self.instructions = scratchwebapi.getMetaDataEntry(self.project_id, "instructions") # self.notes_and_credits = scratchwebapi.getMetaDataEntry(self.project_id, "description") # self.automatic_screenshot_image_url = "{}{}.png".format(scratchwebapi.SCRATCH_PROJECT_IMAGE_BASE_URL, self.project_id) if progress_bar != None: progress_bar.update(ProgressType.DETAILS) # details step passed _log.info("Scratch project: %s%s", self.name, "(ID: {})".format(self.project_id) if self.project_id > 0 else "") self.name = self.name.strip() if self.name != None else "Unknown Project" self.md5_to_resource_path_map = read_md5_to_resource_path_mapping() self.global_user_lists = self.objects[0].get_lists() for scratch_object in self.objects: verify_resources_of_scratch_object(scratch_object, self.md5_to_resource_path_map, self.project_base_path) listened_keys = [] for scratch_obj in self.objects: for script in scratch_obj.scripts: if script.type == SCRIPT_KEY_PRESSED: assert len(script.arguments) == 1 listened_keys += [(argument, "listenedKeys") for argument in script.arguments] try: self.listened_keys.update(listened_keys) except AttributeError: self.listened_keys = set(listened_keys) # TODO: rename self.background_md5_names = set([costume[JsonKeys.COSTUME_MD5] for costume in self.get_costumes()]) result = self.find_unused_resources_name_and_filepath() self.unused_resource_names = result[0] if len(result) > 0 else [] self.unused_resource_paths = result[1] if len(result) > 0 else [] for unused_path in self.unused_resource_paths: _log.warning("Project folder contains unused resource file: '%s'. These " \ "will be omitted for Catrobat project.", os.path.basename(unused_path))
self.send_response_data(response.as_dict()) return except Exception, e: _logger.warn("Unable to download project's web page: " + str(e)) if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(ProjectDataResponse().as_dict()) return if project_html_content is None or project_html_content.body is None \ or not isinstance(project_html_content.body, (str, unicode)): _logger.error("Unable to download web page of project: Invalid or empty HTML-content!") if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(ProjectDataResponse().as_dict()) return visibility_state = scratchwebapi.getMetaDataEntry(project_id , "visibility") response = ProjectDataResponse() response.accessible = True response.visibility_state = visibility_state response.valid_until = dt.now() + timedelta(seconds=cls.CACHE_ENTRY_VALID_FOR) if visibility_state != ScratchProjectVisibiltyState.PUBLIC: _logger.warn("Not allowed to access non-public scratch-project!") cls.RESPONSE_CACHE[project_id] = (response.as_dict(), response.valid_until) if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(response.as_dict()) return project_info = scratchwebapi.extract_project_details(project_id, escape_quotes=True) if project_info is None: _logger.error("Unable to parse project-info from web page: Invalid or empty HTML-content!")
if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(ProjectDataResponse().as_dict()) return if project_html_content is None or project_html_content.body is None \ or not isinstance(project_html_content.body, (str, unicode)): _logger.error( "Unable to download web page of project: Invalid or empty HTML-content!" ) if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(ProjectDataResponse().as_dict()) return visibility_state = scratchwebapi.getMetaDataEntry( project_id, "visibility") response = ProjectDataResponse() response.accessible = True response.visibility_state = visibility_state response.valid_until = dt.now() + timedelta( seconds=cls.CACHE_ENTRY_VALID_FOR) if visibility_state != ScratchProjectVisibiltyState.PUBLIC: _logger.warn("Not allowed to access non-public scratch-project!") cls.RESPONSE_CACHE[project_id] = (response.as_dict(), response.valid_until) if project_id in cls.IN_PROGRESS_FUTURE_MAP: del cls.IN_PROGRESS_FUTURE_MAP[project_id] self.send_response_data(response.as_dict()) return