def test_punt_socket_traffic_multi_ports_single_socket(self): """ Punt socket traffic multi ports and single socket""" for p in self.ports: self.portsCheck[p] = 0 # # create stream with random pakets count per given ports # pkts = list() for _ in range(0, self.nr_packets): # choose port from port list p = random.choice(self.ports) pkts.append(( Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) / IPv6(src=self.pg0.remote_ip6, dst=self.pg0.local_ip6) / inet6.UDP(sport=9876, dport=p) / Raw('\xa5' * 100))) self.portsCheck[p] += 1 # # no punt socket # punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0) # # configure a punt socket # self.socket_client_create(b"%s/socket_multi" % six.ensure_binary(self.tempdir)) for p in self.ports: self.vapi.punt_socket_register(p, b"%s/socket_multi" % six.ensure_binary(self.tempdir), is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), len(self.ports)) for p in self.ports: self.logger.debug("Send %s packets to port %d", str(self.portsCheck[p]), p) # # expect punt socket and no packets on pg0 # self.vapi.cli("clear errors") self.vapi.cli("clear trace") self.pg0.add_stream(pkts) self.pg_enable_capture(self.pg_interfaces) self.pg_start() self.pg0.get_capture(0) self.logger.info(self.vapi.cli("show trace")) self.socket_client_close() for p in self.ports: self.assertEqual(self.portsCheck[p], 0) self.vapi.punt_socket_deregister(p, is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0)
def test_ensure_binary_raise(self): converted_unicode = six.ensure_binary(self.UNICODE_EMOJI, encoding='utf-8', errors='strict') converted_binary = six.ensure_binary(self.BINARY_EMOJI, encoding="utf-8", errors='strict') if six.PY2: # PY2: unicode -> str assert converted_unicode == self.BINARY_EMOJI and isinstance(converted_unicode, str) # PY2: str -> str assert converted_binary == self.BINARY_EMOJI and isinstance(converted_binary, str) else: # PY3: str -> bytes assert converted_unicode == self.BINARY_EMOJI and isinstance(converted_unicode, bytes) # PY3: bytes -> bytes assert converted_binary == self.BINARY_EMOJI and isinstance(converted_binary, bytes)
def add_vpp_config(self): rv = self._test.vapi.memif_socket_filename_add_del( 1, self.socket_id, self.socket_filename) if self.add_default_folder: self.socket_filename = b"%s/%s" % ( six.ensure_binary(self._test.tempdir, encoding='utf-8'), self.socket_filename) return rv
def test_punt_socket_dump(self): """ Punt socket registration """ punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0) # # configure a punt socket # self.vapi.punt_socket_register(1111, b"%s/socket_1111" % six.ensure_binary(self.tempdir), is_ip4=0) self.vapi.punt_socket_register(2222, b"%s/socket_2222" % six.ensure_binary(self.tempdir), is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 2) self.assertEqual(punts[0].punt.l4_port, 1111) self.assertEqual(punts[1].punt.l4_port, 2222) # # deregister a punt socket # self.vapi.punt_socket_deregister(1111, is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 1) # # configure a punt socket again # self.vapi.punt_socket_register(1111, b"%s/socket_1111" % six.ensure_binary(self.tempdir), is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 2) # # deregister all punt socket # self.vapi.punt_socket_deregister(1111, is_ip4=0) self.vapi.punt_socket_deregister(2222, is_ip4=0) self.vapi.punt_socket_deregister(3333, is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0)
def _create_netlogon_query(self, domain, msgid): """Create a netlogon query for `domain'.""" client = ldap.Client() hostname = misc.hostname() filter = '(&(DnsDomain=%s)(Host=%s)(NtVer=\\06\\00\\00\\00))' % \ (domain, hostname) attrs = ('NetLogon',) query = client.create_search_request('', filter, attrs=attrs, scope=ldap.SCOPE_BASE, msgid=msgid) return six.ensure_binary(query)
def _do_create_snapshot(data): snapshot_name = data["snapshot_name"] work_dir = snapshot_dir + "/workdir/%s" % snapshot_name try: if not os.path.exists(work_dir): os.makedirs(work_dir) # Open / initialize files filename_target = "%s/%s" % (snapshot_dir, snapshot_name) filename_work = "%s/%s.work" % (work_dir, snapshot_name) open(filename_target, "w").close() def get_basic_tarinfo(name): tarinfo = tarfile.TarInfo(name) tarinfo.mtime = time.time() tarinfo.uid = 0 tarinfo.gid = 0 tarinfo.mode = 0o644 tarinfo.type = tarfile.REGTYPE return tarinfo # Initialize the snapshot tar file and populate with initial information tar_in_progress = tarfile.open(filename_work, "w") for key in ["comment", "created_by", "type"]: tarinfo = get_basic_tarinfo(key) encoded_value = data[key].encode("utf-8") tarinfo.size = len(encoded_value) tar_in_progress.addfile(tarinfo, io.BytesIO(encoded_value)) tar_in_progress.close() # Process domains (sorted) subtar_info = {} for name, info in sorted(_get_default_backup_domains().items()): prefix = info.get("prefix", "") filename_subtar = "%s.tar.gz" % name path_subtar = "%s/%s" % (work_dir, filename_subtar) paths = ["." if x[1] == "" else x[1] for x in info.get("paths", [])] command = [ "tar", "czf", path_subtar, "--ignore-failed-read", "--force-local", "-C", prefix ] + paths proc = subprocess.Popen( command, stdin=None, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=prefix, encoding="utf-8", ) _stdout, stderr = proc.communicate() exit_code = proc.wait() # Allow exit codes 0 and 1 (files changed during backup) if exit_code not in [0, 1]: raise MKGeneralException( "Error while creating backup of %s (Exit Code %d) - %s.\n%s" % (name, exit_code, stderr, command)) subtar_hash = sha256(open(path_subtar, "rb").read()).hexdigest() subtar_signed = sha256(six.ensure_binary(subtar_hash) + _snapshot_secret()).hexdigest() subtar_info[filename_subtar] = (subtar_hash, subtar_signed) # Append tar.gz subtar to snapshot command = ["tar", "--append", "--file=" + filename_work, filename_subtar] proc = subprocess.Popen( command, cwd=work_dir, close_fds=True, ) proc.communicate() exit_code = proc.wait() if os.path.exists(filename_subtar): os.unlink(filename_subtar) if exit_code != 0: raise MKGeneralException("Error on adding backup domain %s to tarfile" % name) # Now add the info file which contains hashes and signed hashes for # each of the subtars info = ''.join(['%s %s %s\n' % (k, v[0], v[1]) for k, v in subtar_info.items()]) + '\n' tar_in_progress = tarfile.open(filename_work, "a") tarinfo = get_basic_tarinfo("checksums") tarinfo.size = len(info) tar_in_progress.addfile(tarinfo, io.BytesIO(info)) tar_in_progress.close() shutil.move(filename_work, filename_target) finally: shutil.rmtree(work_dir)
def add_file(self, name, kind=tarfile.REGTYPE, content=None, link=None, file_content=None, uid=0, gid=0, uname='', gname='', mtime=None, mode=None): """Add a file to the current tar. Args: name: the name of the file to add. kind: the type of the file to add, see tarfile.*TYPE. content: a textual content to put in the file. link: if the file is a link, the destination of the link. file_content: file to read the content from. Provide either this one or `content` to specifies a content for the file. uid: owner user identifier. gid: owner group identifier. uname: owner user names. gname: owner group names. mtime: modification time to put in the archive. mode: unix permission mode of the file, default 0644 (0755). """ if file_content and os.path.isdir(file_content): # Recurse into directory self.add_dir(name, file_content, uid, gid, uname, gname, mtime, mode) return if not (name == self.root_directory or name.startswith('/') or name.startswith(self.root_directory + '/')): name = os.path.join(self.root_directory, name) if kind == tarfile.DIRTYPE: name = name.rstrip('/') if name in self.directories: return if mtime is None: mtime = self.default_mtime components = name.rsplit('/', 1) if len(components) > 1: d = components[0] self.add_file(d, tarfile.DIRTYPE, uid=uid, gid=gid, uname=uname, gname=gname, mtime=mtime, mode=0o755) tarinfo = tarfile.TarInfo(name) tarinfo.mtime = mtime tarinfo.uid = uid tarinfo.gid = gid tarinfo.uname = uname tarinfo.gname = gname tarinfo.type = kind if mode is None: tarinfo.mode = 0o644 if kind == tarfile.REGTYPE else 0o755 else: tarinfo.mode = mode if link: tarinfo.linkname = link if content: content_bytes = six.ensure_binary(content, 'utf-8') tarinfo.size = len(content_bytes) self._addfile(tarinfo, io.BytesIO(content_bytes)) elif file_content: with open(file_content, 'rb') as f: tarinfo.size = os.fstat(f.fileno()).st_size self._addfile(tarinfo, f) else: if kind == tarfile.DIRTYPE: self.directories.add(name) self._addfile(tarinfo)
def test_punt_socket_traffic_single_port_single_socket(self): """ Punt socket traffic single port single socket""" port = self.ports[0] p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) / IPv6(src=self.pg0.remote_ip6, dst=self.pg0.local_ip6) / inet6.UDP(sport=9876, dport=port) / Raw('\xa5' * 100)) pkts = p * self.nr_packets self.portsCheck[port] = self.nr_packets punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0) # # expect ICMPv6 - destination unreachable for all packets # self.vapi.cli("clear trace") self.pg0.add_stream(pkts) self.pg_enable_capture(self.pg_interfaces) self.pg_start() # FIXME - when punt socket deregister is implemented # rx = self.pg0.get_capture(self.nr_packets) # for p in rx: # self.assertEqual(int(p[IPv6].nh), 58) # ICMPv6 # self.assertEqual(int(p[ICMPv6DestUnreach].code),4) # unreachable # # configure a punt socket # self.socket_client_create(b"%s/socket_%d" % ( six.ensure_binary(self.tempdir), port)) self.vapi.punt_socket_register(port, b"%s/socket_%d" % ( six.ensure_binary(self.tempdir), port), is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 1) self.logger.debug("Sending %s packets to port %d", str(self.portsCheck[port]), port) # # expect punt socket and no packets on pg0 # self.vapi.cli("clear errors") self.vapi.cli("clear trace") self.pg0.add_stream(pkts) self.pg_enable_capture(self.pg_interfaces) self.pg_start() self.pg0.get_capture(0) self.logger.info(self.vapi.cli("show trace")) self.socket_client_close() self.assertEqual(self.portsCheck[port], 0) # # remove punt socket. expect ICMP - dest. unreachable for all packets # self.vapi.punt_socket_deregister(port, is_ip4=0) punts = self.vapi.punt_socket_dump(is_ip6=1) self.assertEqual(len(punts), 0) self.pg0.add_stream(pkts) self.pg_enable_capture(self.pg_interfaces) self.pg_start()
def helm_chart(kubeconfig, cluster, docker_image, image_pull_policy): # pylint: disable=redefined-outer-name,unused-argument print('--- \033[32m:helm: Installing Helm chart\033[0m') # Install helm chart try: check_output('''kubectl create namespace dagster-test''', shell=True) print( 'Creating k8s test objects ConfigMap test-env-configmap and Secret test-env-secret' ) config.load_kube_config(config_file=kubeconfig) kube_api = client.CoreV1Api() configmap = client.V1ConfigMap( api_version='v1', kind='ConfigMap', data={'TEST_ENV_VAR': 'foobar'}, metadata=client.V1ObjectMeta(name='test-env-configmap'), ) kube_api.create_namespaced_config_map(namespace='dagster-test', body=configmap) # Secret values are expected to be base64 encoded secret_val = six.ensure_str( base64.b64encode(six.ensure_binary('foobar'))) secret = client.V1Secret( api_version='v1', kind='Secret', data={'TEST_SECRET_ENV_VAR': secret_val}, metadata=client.V1ObjectMeta(name='test-env-secret'), ) kube_api.create_namespaced_secret(namespace='dagster-test', body=secret) try: repository, tag = docker_image.split(':') helm_config = { 'imagePullPolicy': image_pull_policy, 'dagit': { 'image': { 'repository': repository, 'tag': tag }, 'env': { 'TEST_SET_ENV_VAR': 'test_dagit_env_var' }, 'env_config_maps': ['test-env-configmap'], 'env_secrets': ['test-env-secret'], }, 'job_runner': { 'image': { 'repository': repository, 'tag': tag }, 'env': { 'TEST_SET_ENV_VAR': 'test_job_runner_env_var' }, 'env_config_maps': ['test-env-configmap'], 'env_secrets': ['test-env-secret'], }, 'serviceAccount': { 'name': 'dagit-admin' }, 'postgresqlPassword': '******', 'postgresqlDatabase': 'test', 'postgresqlUser': '******', 'celery': { 'extraWorkerQueues': [ { 'name': 'extra-queue-1', 'replicaCount': 1 }, { 'name': 'extra-queue-2', 'replicaCount': 2 }, ] }, } helm_config_yaml = yaml.dump(helm_config, default_flow_style=False) helm_cmd = [ 'helm', 'install', '--namespace', 'dagster-test', '-f', '-', 'dagster', 'helm/dagster/', ] print('Running Helm Install: \n', helm_cmd, '\nwith config:\n', helm_config_yaml) p = subprocess.Popen( helm_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = p.communicate(six.ensure_binary(helm_config_yaml)) print('\n\nstdout:\n', six.ensure_str(stdout)) print('\n\nstderr:\n', six.ensure_str(stderr)) assert p.returncode == 0 check_output( helm_cmd, shell=True, cwd=os.path.join(git_repository_root(), 'python_modules/libraries/dagster-k8s/'), ) # Wait for Dagit pod to be ready (won't actually stay up w/out js rebuild) success, _ = wait_for_pod('dagit') assert success # Wait for additional Celery worker queues to become ready pods = kube_api.list_namespaced_pod(namespace='dagster-test') for extra_queue in helm_config['celery']['extraWorkerQueues']: pod_names = [ p.metadata.name for p in pods.items if extra_queue['name'] in p.metadata.name ] assert len(pod_names) == extra_queue['replicaCount'] for pod in pod_names: success, _ = wait_for_pod(pod) assert success yield finally: print('Uninstalling helm chart') check_output( ['helm uninstall dagster --namespace dagster-test'], shell=True, cwd=os.path.join(git_repository_root(), 'python_modules/libraries/dagster-k8s/'), ) finally: if not IS_BUILDKITE: print('Deleting namespace') check_output( ['kubectl delete namespace dagster-test'], shell=True, ) print('Deleted namespace')
def _run_deprecated_conversion_binary(model_flags_str, conversion_flags_str, input_data_str, debug_info_str=None): """Convert `input_data_str` using deprecated conversion binary. Args: model_flags_str: Serialized proto describing model properties, see `model_flags.proto`. conversion_flags_str: Serialized proto describing TFLite converter properties, see `toco/toco_flags.proto`. input_data_str: Input data in serialized form (e.g. a graphdef is common) debug_info_str: Serialized `GraphDebugInfo` proto describing logging information. (default None) Returns: Converted model in serialized form (e.g. a TFLITE model is common). Raises: ConverterError: When cannot find the deprecated conversion binary. RuntimeError: When conversion fails, an exception is raised with the error message embedded. """ if distutils.spawn.find_executable(_deprecated_conversion_binary) is None: raise ConverterError("""Could not find `toco_from_protos` binary, make sure your virtualenv bin directory or pip local bin directory is in your path. In particular, if you have installed TensorFlow with --user, make sure you add the install directory to your path. For example: Linux: export PATH=$PATH:~/.local/bin/ Mac: export PATH=$PATH:~/Library/Python/<version#>/bin Alternative, use virtualenv.""") # Windows and TemporaryFile are not that useful together, # since you cannot have two readers/writers. So we have to # make the temporaries and close and delete them explicitly. conversion_filename, model_filename, input_filename, output_filename = (None, None, None, None) try: # Build all input files with _tempfile.NamedTemporaryFile(delete=False) as fp_conversion, \ _tempfile.NamedTemporaryFile(delete=False) as fp_model, \ _tempfile.NamedTemporaryFile(delete=False) as fp_input, \ _tempfile.NamedTemporaryFile(delete=False) as fp_debug: conversion_filename = fp_conversion.name input_filename = fp_input.name model_filename = fp_model.name debug_filename = fp_debug.name fp_model.write(model_flags_str) fp_conversion.write(conversion_flags_str) fp_input.write(six.ensure_binary(input_data_str)) debug_info_str = debug_info_str if debug_info_str else "" # if debug_info_str contains a "string value", then the call to # fp_debug.write(debug_info_str) will fail with the following error # # TypeError: a bytes-like object is required, not 'str' # # Some of the subtests within the "convert_test" unit-test fail # with the error shown above. So watch out for that scenario and # convert debug_info_str to bytes where needed if not isinstance(debug_info_str, bytes): fp_debug.write(debug_info_str.encode("utf-8")) else: fp_debug.write(debug_info_str) # Reserve an output file with _tempfile.NamedTemporaryFile(delete=False) as fp: output_filename = fp.name # Run cmd = [ _deprecated_conversion_binary, model_filename, conversion_filename, input_filename, output_filename, "--debug_proto_file={}".format(debug_filename), ] cmdline = " ".join(cmd) is_windows = _platform.system() == "Windows" proc = _subprocess.Popen( cmdline, shell=True, stdout=_subprocess.PIPE, stderr=_subprocess.STDOUT, close_fds=not is_windows) stdout, stderr = proc.communicate() exitcode = proc.returncode if exitcode == 0: with open(output_filename, "rb") as fp: return fp.read() else: stdout = _try_convert_to_unicode(stdout) stderr = _try_convert_to_unicode(stderr) raise ConverterError("See console for info.\n%s\n%s\n" % (stdout, stderr)) finally: # Must manually cleanup files. for filename in [ conversion_filename, input_filename, model_filename, output_filename ]: try: _os.unlink(filename) except (OSError, TypeError): pass
def mk_repr(x: Any) -> bytes: return base64.b64encode(ensure_binary(repr(x)))
def harvest_objects_import(context, data_dict): ''' Reimports the existing harvest objects, specified by either source_id, harvest_object_id or package_id. It performs the import stage with the last fetched objects, optionally belonging to a certain source. Please note that no objects will be fetched from the remote server. It will only affect the last fetched objects already present in the database. :param source_id: the id of the harvest source to import :type source_id: string :param guid: the guid of the harvest object to import :type guid: string :param harvest_object_id: the id of the harvest object to import :type harvest_object_id: string :param package_id: the id or name of the package to import :type package_id: string ''' log.info('Harvest objects import: %r', data_dict) check_access('harvest_objects_import', context, data_dict) session = context['session'] source_id = data_dict.get('source_id') guid = data_dict.get('guid') harvest_object_id = data_dict.get('harvest_object_id') package_id_or_name = data_dict.get('package_id') segments = context.get('segments') join_datasets = context.get('join_datasets', True) if guid: last_objects_ids = \ session.query(HarvestObject.id) \ .filter(HarvestObject.guid == guid) \ .filter(HarvestObject.current == True) # noqa: E712 elif source_id: source = HarvestSource.get(source_id) if not source: log.error('Harvest source %s does not exist', source_id) raise NotFound('Harvest source %s does not exist' % source_id) if not source.active: log.warn('Harvest source %s is not active.', source_id) raise Exception('This harvest source is not active') last_objects_ids = \ session.query(HarvestObject.id) \ .join(HarvestSource) \ .filter(HarvestObject.source == source) \ .filter(HarvestObject.current == True) # noqa: E712 elif harvest_object_id: last_objects_ids = \ session.query(HarvestObject.id) \ .filter(HarvestObject.id == harvest_object_id) elif package_id_or_name: last_objects_ids = ( session.query(HarvestObject.id).join(Package).filter( HarvestObject.current == True) # noqa: E712 .filter(Package.state == u'active').filter( or_(Package.id == package_id_or_name, Package.name == package_id_or_name))) join_datasets = False else: last_objects_ids = \ session.query(HarvestObject.id) \ .filter(HarvestObject.current == True) # noqa: E712 if join_datasets: last_objects_ids = last_objects_ids.join(Package) \ .filter(Package.state == u'active') last_objects_ids = last_objects_ids.all() last_objects_count = 0 for obj_id in last_objects_ids: if segments and \ str(hashlib.md5(six.ensure_binary(obj_id[0])).hexdigest())[0] not in segments: continue obj = session.query(HarvestObject).get(obj_id) for harvester in PluginImplementations(IHarvester): if harvester.info()['name'] == obj.source.type: if hasattr(harvester, 'force_import'): harvester.force_import = True harvester.import_stage(obj) break last_objects_count += 1 log.info('Harvest objects imported: %s', last_objects_count) return last_objects_count
def check_server(force=False, change_user=False, notify=False): log.debug("checkServer Called") settings = xbmcaddon.Addon() server_url = "" something_changed = False du = DownloadUtils() if force is False: # if not forcing use server details from settings svr = du.get_server() if svr is not None: server_url = svr # if the server is not set then try to detect it if server_url == "": # scan for local server server_info = get_server_details() addon = xbmcaddon.Addon() server_icon = addon.getAddonInfo('icon') server_list = [] for server in server_info: server_item = xbmcgui.ListItem( server.get("Name", string_load(30063))) sub_line = server.get("Address") server_item.setLabel2(sub_line) server_item.setProperty("address", server.get("Address")) art = {"Thumb": server_icon} server_item.setArt(art) server_list.append(server_item) if len(server_list) > 0: return_index = xbmcgui.Dialog().select('{} : {}'.format( __addon_name__, string_load(30166)), server_list, useDetails=True) if return_index != -1: server_url = server_info[return_index]["Address"] if not server_url: return_index = xbmcgui.Dialog().yesno( __addon_name__, '{}\n{}'.format(string_load(30282), string_load(30370))) if not return_index: xbmc.executebuiltin("ActivateWindow(Home)") return while True: kb = xbmc.Keyboard() kb.setHeading(string_load(30372)) if server_url: kb.setDefault(server_url) else: kb.setDefault("http://<server address>:8096") kb.doModal() if kb.isConfirmed(): server_url = kb.getText() else: xbmc.executebuiltin("ActivateWindow(Home)") return public_lookup_url = "%s/System/Info/Public?format=json" % ( server_url) log.debug("Testing_Url: {0}".format(public_lookup_url)) progress = xbmcgui.DialogProgress() progress.create('{} : {}'.format(__addon_name__, string_load(30376))) progress.update(0, string_load(30377)) result = du.download_url(public_lookup_url, authenticate=False) progress.close() if result: xbmcgui.Dialog().ok( '{} : {}'.format(__addon_name__, string_load(30167)), server_url) break else: return_index = xbmcgui.Dialog().yesno( '{} : {}'.format(__addon_name__, string_load(30135)), server_url, string_load(30371)) if not return_index: xbmc.executebuiltin("ActivateWindow(Home)") return log.debug("Selected server: {0}".format(server_url)) settings.setSetting("server_address", server_url) something_changed = True # do we need to change the user user_details = load_user_details(settings) current_username = user_details.get("username", "") current_username = py2_decode(current_username) # if asked or we have no current user then show user selection screen if something_changed or change_user or len(current_username) == 0: # stop playback when switching users xbmc.Player().stop() du = DownloadUtils() # get a list of users log.debug("Getting user list") result = du.download_url(server_url + "/Users/Public?format=json", authenticate=False) log.debug("jsonData: {0}".format(py2_decode(result))) selected_id = -1 users = [] for user in result: config = user.get("Configuration") if config is not None: if config.get("IsHidden", False) is False: name = user.get("Name") admin = user.get("Policy", {}).get("IsAdministrator", False) time_ago = "" last_active = user.get("LastActivityDate") if last_active: last_active_date = datetime_from_string(last_active) log.debug( "LastActivityDate: {0}".format(last_active_date)) ago = datetime.now() - last_active_date log.debug("LastActivityDate: {0}".format(ago)) days = divmod(ago.seconds, 86400) hours = divmod(days[1], 3600) minutes = divmod(hours[1], 60) log.debug("LastActivityDate: {0} {1} {2}".format( days[0], hours[0], minutes[0])) if days[0]: time_ago += " %sd" % days[0] if hours[0]: time_ago += " %sh" % hours[0] if minutes[0]: time_ago += " %sm" % minutes[0] time_ago = time_ago.strip() if not time_ago: time_ago = "Active: now" else: time_ago = "Active: %s ago" % time_ago log.debug("LastActivityDate: {0}".format(time_ago)) user_item = xbmcgui.ListItem(name) user_image = du.get_user_artwork(user, 'Primary') if not user_image: user_image = "DefaultUser.png" art = {"Thumb": user_image} user_item.setArt(art) user_item.setLabel2("TEST") sub_line = time_ago if user.get("HasPassword", False) is True: sub_line += ", Password" user_item.setProperty("secure", "true") m = hashlib.md5() m.update(ensure_binary(name)) hashed_username = m.hexdigest() saved_password = settings.getSetting( "saved_user_password_" + hashed_username) if saved_password: sub_line += ": Saved" else: user_item.setProperty("secure", "false") if admin: sub_line += ", Admin" else: sub_line += ", User" user_item.setProperty("manual", "false") user_item.setLabel2(sub_line) users.append(user_item) if current_username == name: selected_id = len(users) - 1 if current_username: selection_title = string_load( 30180) + " (" + current_username + ")" else: selection_title = string_load(30180) # add manual login user_item = xbmcgui.ListItem(string_load(30365)) art = {"Thumb": "DefaultUser.png"} user_item.setArt(art) user_item.setLabel2(string_load(30366)) user_item.setProperty("secure", "true") user_item.setProperty("manual", "true") users.append(user_item) return_value = xbmcgui.Dialog().select(selection_title, users, preselect=selected_id, autoclose=20000, useDetails=True) if return_value > -1 and return_value != selected_id: something_changed = True selected_user = users[return_value] secured = selected_user.getProperty("secure") == "true" manual = selected_user.getProperty("manual") == "true" selected_user_name = selected_user.getLabel() log.debug("Selected User Name: {0} : {1}".format( return_value, selected_user_name)) if manual: kb = xbmc.Keyboard() kb.setHeading(string_load(30005)) if current_username: kb.setDefault(current_username) kb.doModal() if kb.isConfirmed(): selected_user_name = kb.getText() log.debug("Manual entered username: {0}".format( selected_user_name)) else: return if secured: # we need a password, check the settings first m = hashlib.md5() m.update(selected_user_name.encode()) hashed_username = m.hexdigest() saved_password = settings.getSetting("saved_user_password_" + hashed_username) allow_password_saving = settings.getSetting( "allow_password_saving") == "true" # if not saving passwords but have a saved ask to clear it if not allow_password_saving and saved_password: clear_password = xbmcgui.Dialog().yesno( string_load(30368), string_load(30369)) if clear_password: settings.setSetting( "saved_user_password_" + hashed_username, "") if saved_password: log.debug("Saving username and password: {0}".format( selected_user_name)) log.debug("Using stored password for user: {0}".format( hashed_username)) save_user_details(settings, selected_user_name, saved_password) else: kb = xbmc.Keyboard() kb.setHeading(string_load(30006)) kb.setHiddenInput(True) kb.doModal() if kb.isConfirmed(): log.debug("Saving username and password: {0}".format( selected_user_name)) save_user_details(settings, selected_user_name, kb.getText()) # should we save the password if allow_password_saving: save_password = xbmcgui.Dialog().yesno( string_load(30363), string_load(30364)) if save_password: log.debug( "Saving password for fast user switching: {0}" .format(hashed_username)) settings.setSetting( "saved_user_password_" + hashed_username, kb.getText()) else: log.debug("Saving username with no password: {0}".format( selected_user_name)) save_user_details(settings, selected_user_name, "") if something_changed: home_window = HomeWindow() home_window.clear_property("userid") home_window.clear_property("AccessToken") home_window.clear_property("userimage") home_window.clear_property("embycon_widget_reload") du = DownloadUtils() du.authenticate() du.get_user_id() xbmc.executebuiltin("ActivateWindow(Home)") if "estuary_embycon" in xbmc.getSkinDir(): xbmc.executebuiltin("SetFocus(9000, 0, absolute)") xbmc.executebuiltin("ReloadSkin()")
def mk_repr(x): # type: (Any) -> bytes r = pickle.dumps(x) if config.wato_legacy_eval else six.ensure_binary(repr(x)) return base64.b64encode(r)
def NoMoredata(self, data): if self.IsAlive: if self.action == "listall": self.request.setHeader("content-type", "application/json; charset=utf-8") try: data = self.parseAll() self.request.write(six.ensure_binary(json.dumps(data))) except Exception as exc: self.request.setResponseCode(http.INTERNAL_SERVER_ERROR) self.request.write( six.ensure_binary( json.dumps({ "result": False, "request": self.request.path, "exception": repr(exc) }))) elif self.action == "full": try: data = open("/tmp/opkg.tmp", 'r').read() self.request.write(six.ensure_binary(data)) except Exception as exc: self.request.setResponseCode(http.INTERNAL_SERVER_ERROR) self.request.write(six.ensure_binary(repr(exc))) else: nresult = "" if self.action == "list": for a in self.ResultString.split("\n"): if a.count(" - ") > 0: nresult += a + "\n" else: for a in self.ResultString.split("\n"): if a.count(" - ") > 0: if nresult[:-1] == "\n": nresult += a else: nresult += "\n" + a else: nresult += a + "\n" nresult = nresult.replace("\n\n", "\n") nresult = nresult.replace("\n ", " ") if self.filter is not None: pl = nresult.split("\n") add = True rpl = [] for p in pl: if p.count(" - ") > 0: add = True name = p.split(' - ')[0] for f in self.filter: if name.endswith('-' + f): add = False if add: rpl.append(p) if self.json: data = [] data.append({"result": True, "packages": rpl}) self.request.setHeader("content-type", "application/json; charset=utf-8") self.request.write(six.ensure_binary(json.dumps(data))) else: nresult = '\n'.join(rpl) nresult.replace('\n', '<br>\n') nresult = six.ensure_binary(nresult) self.request.write(b"<html><body>\n") self.request.write(nresult) self.request.write(b"</body></html>\n") self.request.finish() return server.NOT_DONE_YET
def ShowError(self, request, text): request.setResponseCode(http.OK) request.write(six.ensure_binary(text)) request.finish() return server.NOT_DONE_YET
def validate_mtime(self, mtime, path): return self.packet( TRANSLATE_VALIDATE_MTIME, struct.pack('L', int(mtime)) + six.ensure_binary(path))
def expand_request_header(self, name, value): assert isinstance(name, (str, bytes)) assert isinstance(value, (str, bytes)) return self.packet( TRANSLATE_EXPAND_REQUEST_HEADER, six.ensure_binary(name) + b':' + six.ensure_binary(value))
def response_header(self, name, value): assert isinstance(name, (str, bytes)) assert isinstance(value, (str, bytes)) return self.packet( TRANSLATE_REQUEST_HEADER, six.ensure_binary(name) + b':' + six.ensure_binary(value))
def doMerge(outputio, files, authoropts=[], titleopt=None, descopt=None, tags=[], languages=['en'], titlenavpoints=True, originalnavpoints=True, flattentoc=False, printtimes=False, coverjpgpath=None, keepmetadatafiles=False, source=None): ''' outputio = output file name or BytesIO. files = list of input file names or BytesIOs. authoropts = list of authors to use, otherwise add from all input titleopt = title, otherwise '<first title> Anthology' descopt = description, otherwise '<title> by <author>' list for all input tags = dc:subject tags to include, otherwise none. languages = dc:language tags to include titlenavpoints if true, put in a new TOC entry for each epub, nesting each epub's chapters under it originalnavpoints if true, include the original TOCs from each epub flattentoc if true, flatten TOC down to one level only. coverjpgpath, Path to a jpg to use as cover image. ''' printt = partial(cond_print,printtimes) ## Python 2.5 ZipFile is rather more primative than later ## versions. It can operate on a file, or on a BytesIO, but ## not on an open stream. OTOH, I suspect we would have had ## problems with closing and opening again to change the ## compression type anyway. filecount=0 t = time() ## Write mimetype file, must be first and uncompressed. ## Older versions of python(2.4/5) don't allow you to specify ## compression by individual file. ## Overwrite if existing output file. outputepub = ZipFile(outputio, "w", compression=ZIP_STORED, allowZip64=True) outputepub.debug = 3 outputepub.writestr("mimetype", "application/epub+zip") outputepub.close() ## Re-open file for content. outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED, allowZip64=True) outputepub.debug = 3 ## Create META-INF/container.xml file. The only thing it does is ## point to content.opf containerdom = getDOMImplementation().createDocument(None, "container", None) containertop = containerdom.documentElement containertop.setAttribute("version","1.0") containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container") rootfiles = containerdom.createElement("rootfiles") containertop.appendChild(rootfiles) rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf", "media-type":"application/oebps-package+xml"})) outputepub.writestr("META-INF/container.xml",containerdom.toprettyxml(indent=' ',encoding='utf-8')) ## Process input epubs. items = [] # list of (id, href, type) tuples(all strings) -- From .opfs' manifests items.append(("ncx","toc.ncx","application/x-dtbncx+xml")) ## we'll generate the toc.ncx file, ## but it needs to be in the items manifest. itemrefs = [] # list of strings -- idrefs from .opfs' spines navmaps = [] # list of navMap DOM elements -- TOC data for each from toc.ncx files is_ffdl_epub = [] # list of t/f itemhrefs = {} # hash of item[id]s to itemref[href]s -- to find true start of book(s). firstitemhrefs = [] booktitles = [] # list of strings -- Each book's title allauthors = [] # list of lists of strings -- Each book's list of authors. filelist = [] printt("prep output:%s"%(time()-t)) t = time() booknum=1 firstmetadom = None for file in files: if file == None : continue book = "%d" % booknum bookdir = "%d/" % booknum bookid = "a%d" % booknum epub = ZipFile(file, 'r') ## Find the .opf file. container = epub.read("META-INF/container.xml") containerdom = parseString(container) rootfilenodelist = containerdom.getElementsByTagNameNS("*","rootfile") rootfilename = rootfilenodelist[0].getAttribute("full-path") ## Save the path to the .opf file--hrefs inside it are relative to it. relpath = get_path_part(rootfilename) metadom = parseString(epub.read(rootfilename)) # logger.debug("metadom:%s"%epub.read(rootfilename)) if booknum==1 and not source: try: firstmetadom = metadom.getElementsByTagNameNS("*","metadata")[0] source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8") except: source="" # if the epub was ever edited with Sigil, it changed the unique-identifier, # but dc:contributor was left. #is_ffdl_epub.append(metadom.documentElement.getAttribute('unique-identifier') == "fanficdownloader-uid") is_ffdl_epub.append(False) for c in metadom.getElementsByTagName("dc:contributor"): # logger.debug("dc:contributor:%s"%getText(c.childNodes)) if c.getAttribute("opf:role") == "bkp" and \ getText(c.childNodes) in ["fanficdownloader [http://fanficdownloader.googlecode.com]", "FanFicFare [https://github.com/JimmXinu/FanFicFare]"]: is_ffdl_epub[-1] = True # set last. break; ## Save indiv book title try: booktitles.append(metadom.getElementsByTagName("dc:title")[0].firstChild.data) except: booktitles.append("(Title Missing)") ## Save authors. authors=[] for creator in metadom.getElementsByTagName("dc:creator"): try: if( creator.getAttribute("opf:role") == "aut" or not creator.hasAttribute("opf:role") and creator.firstChild != None): authors.append(creator.firstChild.data) except: pass if len(authors) == 0: authors.append("(Author Missing)") allauthors.append(authors) if keepmetadatafiles: itemid=bookid+"rootfile" itemhref = rootfilename href=bookdir+itemhref logger.debug("write rootfile %s to %s"%(itemhref,href)) outputepub.writestr(href, epub.read(itemhref)) items.append((itemid,href,"origrootfile/xml")) # spin through the manifest--only place there are item tags. # Correction--only place there *should* be item tags. But # somebody found one that did. manifesttag=metadom.getElementsByTagNameNS("*","manifest")[0] for item in manifesttag.getElementsByTagNameNS("*","item"): itemid=bookid+item.getAttribute("id") itemhref = normpath(unquote(item.getAttribute("href"))) # remove %20, etc. href=bookdir+relpath+itemhref # if item.getAttribute("properties") == "nav": # # epub3 TOC file is only one with this type--as far as I know. # # grab the whole navmap, deal with it later. # el if item.getAttribute("media-type") == "application/x-dtbncx+xml": # epub2 TOC file is only one with this type--as far as I know. # grab the whole navmap, deal with it later. tocdom = parseString(epub.read(normpath(relpath+item.getAttribute("href")))) # update all navpoint ids with bookid for uniqueness. for navpoint in tocdom.getElementsByTagNameNS("*","navPoint"): navpoint.setAttribute("id",bookid+navpoint.getAttribute("id")) # update all content paths with bookdir for uniqueness. for content in tocdom.getElementsByTagNameNS("*","content"): content.setAttribute("src",normpath(bookdir+relpath+content.getAttribute("src"))) navmaps.append(tocdom.getElementsByTagNameNS("*","navMap")[0]) if keepmetadatafiles: logger.debug("write toc.ncx %s to %s"%(relpath+itemhref,href)) outputepub.writestr(href, epub.read(normpath(relpath+itemhref))) items.append((itemid,href,"origtocncx/xml")) else: #href=href.encode('utf8') logger.debug("item id: %s -> %s:"%(itemid,href)) itemhrefs[itemid] = href if href not in filelist: try: outputepub.writestr(href, epub.read(normpath(relpath+itemhref))) if re.match(r'.*/(file|chapter)\d+\.x?html',href): filecount+=1 items.append((itemid,href,item.getAttribute("media-type"))) filelist.append(href) except KeyError as ke: # Skip missing files. logger.info("Skipping missing file %s (%s)"%(href,relpath+itemhref)) del itemhrefs[itemid] itemreflist = metadom.getElementsByTagNameNS("*","itemref") # logger.debug("itemhrefs:%s"%itemhrefs) logger.debug("bookid:%s"%bookid) logger.debug("itemreflist[0].getAttribute(idref):%s"%itemreflist[0].getAttribute("idref")) # Looking for the first item in itemreflist that wasn't # discarded due to missing files. for itemref in itemreflist: idref = bookid+itemref.getAttribute("idref") if idref in itemhrefs: firstitemhrefs.append(itemhrefs[idref]) break for itemref in itemreflist: itemrefs.append(bookid+itemref.getAttribute("idref")) # logger.debug("adding to itemrefs:%s"%itemref.toprettyxml()) booknum=booknum+1; printt("after file loop:%s"%(time()-t)) t = time() ## create content.opf file. uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme. contentdom = getDOMImplementation().createDocument(None, "package", None) package = contentdom.documentElement package.setAttribute("version","2.0") package.setAttribute("xmlns","http://www.idpf.org/2007/opf") package.setAttribute("unique-identifier","epubmerge-id") metadata=newTag(contentdom,"metadata", attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/", "xmlns:opf":"http://www.idpf.org/2007/opf"}) metadata.appendChild(newTag(contentdom,"dc:identifier",text=uniqueid,attrs={"id":"epubmerge-id"})) if( titleopt is None ): titleopt = booktitles[0]+" Anthology" metadata.appendChild(newTag(contentdom,"dc:title",text=titleopt)) # If cmdline authors, use those instead of those collected from the epubs # (allauthors kept for TOC & description gen below. if( len(authoropts) > 1 ): useauthors=[authoropts] else: useauthors=allauthors usedauthors=dict() for authorlist in useauthors: for author in authorlist: if( author not in usedauthors ): usedauthors[author]=author metadata.appendChild(newTag(contentdom,"dc:creator", attrs={"opf:role":"aut"}, text=author)) metadata.appendChild(newTag(contentdom,"dc:contributor",text="epubmerge",attrs={"opf:role":"bkp"})) metadata.appendChild(newTag(contentdom,"dc:rights",text="Copyrights as per source stories")) for l in languages: metadata.appendChild(newTag(contentdom,"dc:language",text=l)) if not descopt: # created now, but not filled in until TOC generation to save loops. description = newTag(contentdom,"dc:description",text="Anthology containing:\n") else: description = newTag(contentdom,"dc:description",text=descopt) metadata.appendChild(description) if source: metadata.appendChild(newTag(contentdom,"dc:identifier", attrs={"opf:scheme":"URL"}, text=source)) metadata.appendChild(newTag(contentdom,"dc:source", text=source)) for tag in tags: metadata.appendChild(newTag(contentdom,"dc:subject",text=tag)) package.appendChild(metadata) manifest = contentdom.createElement("manifest") package.appendChild(manifest) spine = newTag(contentdom,"spine",attrs={"toc":"ncx"}) package.appendChild(spine) if coverjpgpath: # in case coverjpg isn't a jpg: coverext = 'jpg' covertype = 'image/jpeg' try: coverext = coverjpgpath.split('.')[-1].lower() covertype = imagetypes.get(coverext,covertype) except: pass logger.debug("coverjpgpath:%s coverext:%s covertype:%s"%(coverjpgpath,coverext,covertype)) # <meta name="cover" content="cover.jpg"/> metadata.appendChild(newTag(contentdom,"meta",{"name":"cover", "content":"coverimageid"})) guide = newTag(contentdom,"guide") guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover", "title":"Cover", "href":"cover.xhtml"})) package.appendChild(guide) manifest.appendChild(newTag(contentdom,"item", attrs={'id':"coverimageid", 'href':"cover."+coverext, 'media-type':covertype})) # Note that the id of the cover xhmtl *must* be 'cover' # for it to work on Nook. manifest.appendChild(newTag(contentdom,"item", attrs={'id':"cover", 'href':"cover.xhtml", 'media-type':"application/xhtml+xml"})) spine.appendChild(newTag(contentdom,"itemref", attrs={"idref":"cover", "linear":"yes"})) for item in items: # logger.debug("new item:%s %s %s"%item) (id,href,type)=item manifest.appendChild(newTag(contentdom,"item", attrs={'id':id, 'href':href, 'media-type':type})) for itemref in itemrefs: # logger.debug("itemref:%s"%itemref) spine.appendChild(newTag(contentdom,"itemref", attrs={"idref":itemref, "linear":"yes"})) ## create toc.ncx file tocncxdom = getDOMImplementation().createDocument(None, "ncx", None) ncx = tocncxdom.documentElement ncx.setAttribute("version","2005-1") ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/") head = tocncxdom.createElement("head") ncx.appendChild(head) head.appendChild(newTag(tocncxdom,"meta", attrs={"name":"dtb:uid", "content":uniqueid})) depthnode = newTag(tocncxdom,"meta", attrs={"name":"dtb:depth", "content":"4"}) head.appendChild(depthnode) head.appendChild(newTag(tocncxdom,"meta", attrs={"name":"dtb:totalPageCount", "content":"0"})) head.appendChild(newTag(tocncxdom,"meta", attrs={"name":"dtb:maxPageNumber", "content":"0"})) docTitle = tocncxdom.createElement("docTitle") docTitle.appendChild(newTag(tocncxdom,"text",text=titleopt)) ncx.appendChild(docTitle) tocnavMap = tocncxdom.createElement("navMap") ncx.appendChild(tocnavMap) booknum=0 printt("wrote initial metadata:%s"%(time()-t)) t = time() for navmap in navmaps: depthnavpoints = navmap.getElementsByTagNameNS("*","navPoint") # for checking more than one TOC entry # logger.debug( [ x.toprettyxml() for x in navmap.childNodes ] ) ## only gets top level TOC entries. sub entries carried inside. navpoints = [ x for x in navmap.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ] # logger.debug("len(navpoints):%s"%len(navpoints)) # logger.debug( [ x.toprettyxml() for x in navpoints ] ) newnav = None if titlenavpoints: newnav = newTag(tocncxdom,"navPoint",{"id":"book%03d"%booknum}) navlabel = newTag(tocncxdom,"navLabel") newnav.appendChild(navlabel) # For purposes of TOC titling & desc, use first book author. Skip adding author if only one. if len(usedauthors) > 1: title = booktitles[booknum]+" by "+allauthors[booknum][0] else: title = booktitles[booknum] navlabel.appendChild(newTag(tocncxdom,"text",text=title)) # Find the first 'spine' item's content for the title navpoint. # Many epubs have the first chapter as first navpoint, so we can't just # copy that anymore. newnav.appendChild(newTag(tocncxdom,"content", {"src":firstitemhrefs[booknum]})) # logger.debug("newnav:%s"%newnav.toprettyxml()) tocnavMap.appendChild(newnav) # logger.debug("tocnavMap:%s"%tocnavMap.toprettyxml()) else: newnav = tocnavMap if not descopt and len(allauthors[booknum]) > 0: description.appendChild(contentdom.createTextNode(booktitles[booknum]+" by "+allauthors[booknum][0]+"\n")) # If only one TOC point(total, not top level), or if not # including title nav point, include sub book TOC entries. if originalnavpoints and (len(depthnavpoints) > 1 or not titlenavpoints): for navpoint in navpoints: # logger.debug("navpoint:%s"%navpoint.toprettyxml()) newnav.appendChild(navpoint) navpoint.is_ffdl_epub = is_ffdl_epub[booknum] booknum=booknum+1; # end of navmaps loop. maxdepth = 0 contentsrcs = {} removednodes = [] ## Force strict ordering of playOrder, stripping out some. playorder=0 # logger.debug("tocncxdom:%s"%tocncxdom.toprettyxml()) for navpoint in tocncxdom.getElementsByTagNameNS("*","navPoint"): # logger.debug("navpoint:%s"%navpoint.toprettyxml()) if navpoint in removednodes: continue # need content[src] to compare for dups. epub wants dup srcs to have same playOrder. contentsrc = None for n in navpoint.childNodes: if isinstance(n,Element) and n.tagName == "content": contentsrc = n.getAttribute("src") logger.debug("contentsrc: %s"%contentsrc) break if( contentsrc not in contentsrcs ): parent = navpoint.parentNode try: # if the epub was ever edited with Sigil, it changed # the id, but the file name is the same. if navpoint.is_ffdl_epub and \ ( navpoint.getAttribute("id").endswith('log_page') \ or contentsrc.endswith("log_page.xhtml") ): logger.debug("Doing sibs 'filter' 1") sibs = [ x for x in parent.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ] # if only logpage and one chapter, remove them from TOC and just show story. if len(sibs) == 2: parent.removeChild(navpoint) logger.debug("Removing %s:"% sibs[0].getAttribute("playOrder")) parent.removeChild(sibs[1]) removednodes.append(sibs[1]) except: pass # New src, new number. contentsrcs[contentsrc] = navpoint.getAttribute("id") playorder += 1 navpoint.setAttribute("playOrder","%d" % playorder) logger.debug("playorder:%d:"%playorder) # need to know depth of deepest navpoint for <meta name="dtb:depth" content="2"/> npdepth = 1 dp = navpoint.parentNode while dp and dp.tagName != "navMap": npdepth += 1 dp = dp.parentNode if npdepth > maxdepth: maxdepth = npdepth else: # same content, look for ffdl and title_page and/or single chapter. # easier to just set it now, even if the node gets removed later. navpoint.setAttribute("playOrder","%d" % playorder) logger.debug("playorder:%d:"%playorder) parent = navpoint.parentNode try: # if the epub was ever edited with Sigil, it changed # the id, but the file name is the same. if navpoint.is_ffdl_epub and \ ( navpoint.getAttribute("id").endswith('title_page') \ or contentsrc.endswith("title_page.xhtml") ): parent.removeChild(navpoint) logger.debug("Doing sibs 'filter' 2") sibs = [ x for x in parent.childNodes if isinstance(x,Element) and x.tagName=="navPoint" ] # if only one chapter after removing title_page, remove it too. if len(sibs) == 1: logger.debug("Removing %s:"% sibs[0].getAttribute("playOrder")) parent.removeChild(sibs[0]) removednodes.append(sibs[0]) except: pass if flattentoc: maxdepth = 1 # already have play order and pesky dup/single chapters # removed, just need to flatten. flattocnavMap = tocncxdom.createElement("navMap") for n in tocnavMap.getElementsByTagNameNS("*","navPoint"): flattocnavMap.appendChild(n) ncx.replaceChild(flattocnavMap,tocnavMap) printt("navmap/toc maddess:%s"%(time()-t)) t = time() depthnode.setAttribute("content","%d"%maxdepth) ## content.opf written now due to description being filled in ## during TOC generation to save loops. contentxml = contentdom.toprettyxml(indent=' ',encoding='utf-8') # tweak for brain damaged Nook STR. Nook insists on name before content. contentxml = contentxml.replace(ensure_binary('<meta content="coverimageid" name="cover"/>'), ensure_binary('<meta name="cover" content="coverimageid"/>')) outputepub.writestr("content.opf",contentxml) outputepub.writestr("toc.ncx",tocncxdom.toprettyxml(indent=' ',encoding='utf-8')) printt("wrote opf/ncx files:%s"%(time()-t)) t = time() if coverjpgpath: # write, not write string. Pulling from file. outputepub.write(coverjpgpath,"cover."+coverext) outputepub.writestr("cover.xhtml",''' <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css"> @page {padding: 0pt; margin:0pt} body { text-align: center; padding:0pt; margin: 0pt; } div { margin: 0pt; padding: 0pt; } </style></head><body><div> <img src="cover.'''+coverext+'''" alt="cover"/> </div></body></html> ''') # declares all the files created by Windows. otherwise, when # it runs in appengine, windows unzips the files as 000 perms. for zf in outputepub.filelist: zf.create_system = 0 outputepub.close() printt("closed outputepub:%s"%(time()-t)) t = time() return (source,filecount)
def add(self, name, data, compress=None, mode=None, skip_duplicates=False): """ Add a new member to the jar archive, with the given name and the given data. The compress option indicates how the given data should be compressed (one of JAR_STORED or JAR_DEFLATE), or compressed according to the default defined when creating the JarWriter (None). True and False are allowed values for backwards compatibility, mapping, respectively, to JAR_DEFLATE and JAR_STORED. When the data should be compressed, it is only really compressed if the compressed size is smaller than the uncompressed size. The mode option gives the unix permissions that should be stored for the jar entry. If a duplicated member is found skip_duplicates will prevent raising an exception if set to True. The given data may be a buffer, a file-like instance, a Deflater or a JarFileReader instance. The latter two allow to avoid uncompressing data to recompress it. """ name = mozpath.normsep(six.ensure_text(name)) if name in self._contents and not skip_duplicates: raise JarWriterError("File %s already in JarWriter" % name) if compress is None: compress = self._compress if compress is True: compress = JAR_DEFLATED if compress is False: compress = JAR_STORED if isinstance(data, (JarFileReader, Deflater)) and data.compress == compress: deflater = data else: deflater = Deflater(compress, compress_level=self._compress_level) if isinstance(data, (six.binary_type, six.string_types)): deflater.write(data) elif hasattr(data, "read"): try: data.seek(0) except (UnsupportedOperation, AttributeError): pass deflater.write(data.read()) else: raise JarWriterError("Don't know how to handle %s" % type(data)) # Fill a central directory entry for this new member. entry = JarCdirEntry() entry["creator_version"] = 20 if mode is not None: # Set creator host system (upper byte of creator_version) # to 3 (Unix) so mode is honored when there is one. entry["creator_version"] |= 3 << 8 entry["external_attr"] = (mode & 0xFFFF) << 16 if deflater.compressed: entry["min_version"] = 20 # Version 2.0 supports deflated streams entry["general_flag"] = 2 # Max compression entry["compression"] = deflater.compress else: entry["min_version"] = 10 # Version 1.0 for stored streams entry["general_flag"] = 0 entry["compression"] = JAR_STORED # January 1st, 2010. See bug 592369. entry["lastmod_date"] = ((2010 - 1980) << 9) | (1 << 5) | 1 entry["lastmod_time"] = 0 entry["crc32"] = deflater.crc32 entry["compressed_size"] = deflater.compressed_size entry["uncompressed_size"] = deflater.uncompressed_size entry["filename"] = six.ensure_binary(name) self._contents[name] = entry, deflater.compressed_data
def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str, debug_info_str=None, enable_mlir_converter=False): """Convert `input_data_str` according to model and toco parameters. Unless you know what you are doing consider using the more friendly `tf.compat.v1.lite.toco_convert`. Args: model_flags_str: Serialized proto describing model properties, see `toco/model_flags.proto`. toco_flags_str: Serialized proto describing conversion properties, see `toco/toco_flags.proto`. input_data_str: Input data in serialized form (e.g. a graphdef is common) debug_info_str: Serialized `GraphDebugInfo` proto describing logging information. (default None) enable_mlir_converter: Enables MLIR-based conversion instead of the default TOCO conversion. (default False) Returns: Converted model in serialized form (e.g. a TFLITE model is common). Raises: ConverterError: When conversion fails in TFLiteConverter, usually due to ops not being supported. RuntimeError: When conversion fails, an exception is raised with the error message embedded. """ # TODO(aselle): When toco does not use fatal errors for failure, we can # switch this on. if not _toco_from_proto_bin: try: model_str = wrap_toco.wrapped_toco_convert(model_flags_str, toco_flags_str, input_data_str, debug_info_str, enable_mlir_converter) return model_str except Exception as e: raise ConverterError(str(e)) # Windows and TemporaryFile are not that useful together, # since you cannot have two readers/writers. So we have to # make the temporaries and close and delete them explicitly. toco_filename, model_filename, input_filename, output_filename = (None, None, None, None) try: # Build all input files with _tempfile.NamedTemporaryFile(delete=False) as fp_toco, \ _tempfile.NamedTemporaryFile(delete=False) as fp_model, \ _tempfile.NamedTemporaryFile(delete=False) as fp_input, \ _tempfile.NamedTemporaryFile(delete=False) as fp_debug: toco_filename = fp_toco.name input_filename = fp_input.name model_filename = fp_model.name debug_filename = fp_debug.name fp_model.write(model_flags_str) fp_toco.write(toco_flags_str) fp_input.write(six.ensure_binary(input_data_str)) debug_info_str = debug_info_str if debug_info_str else "" # if debug_info_str contains a "string value", then the call to # fp_debug.write(debug_info_str) will fail with the following error # # TypeError: a bytes-like object is required, not 'str' # # Some of the subtests within the "convert_test" unit-test fail # with the error shown above. So watch out for that scenario and # convert debug_info_str to bytes where needed if not isinstance(debug_info_str, bytes): fp_debug.write(debug_info_str.encode("utf-8")) else: fp_debug.write(debug_info_str) # Reserve an output file with _tempfile.NamedTemporaryFile(delete=False) as fp: output_filename = fp.name # Run cmd = [ _toco_from_proto_bin, model_filename, toco_filename, input_filename, output_filename, "--debug_proto_file={}".format(debug_filename), ] if enable_mlir_converter: cmd.append("--enable_mlir_converter") cmdline = " ".join(cmd) is_windows = _platform.system() == "Windows" proc = _subprocess.Popen(cmdline, shell=True, stdout=_subprocess.PIPE, stderr=_subprocess.STDOUT, close_fds=not is_windows) stdout, stderr = proc.communicate() exitcode = proc.returncode if exitcode == 0: with open(output_filename, "rb") as fp: return fp.read() else: stdout = _try_convert_to_unicode(stdout) stderr = _try_convert_to_unicode(stderr) raise ConverterError("See console for info.\n%s\n%s\n" % (stdout, stderr)) finally: # Must manually cleanup files. for filename in [ toco_filename, input_filename, model_filename, output_filename ]: try: _os.unlink(filename) except (OSError, TypeError): pass
def render(self, request): action = getUrlArg(request, "action", "download") file = getUrlArg(request, "file") if file != None: filename = lenient_force_utf_8(file) filename = sanitise_filename_slashes(os.path.realpath(filename)) if not os.path.exists(filename): return "File '%s' not found" % (filename) if action == "stream": name = getUrlArg(request, "name", "stream") port = config.OpenWebif.port.value proto = 'http' if request.isSecure(): port = config.OpenWebif.https_port.value proto = 'https' ourhost = request.getHeader('host') m = re.match('.+\:(\d+)$', ourhost) if m is not None: port = m.group(1) response = "#EXTM3U\n#EXTVLCOPT--http-reconnect=true\n#EXTINF:-1,%s\n%s://%s:%s/file?action=download&file=%s" % ( name, proto, request.getRequestHostname(), port, quote(filename)) request.setHeader("Content-Disposition", 'attachment;filename="%s.m3u"' % name) request.setHeader("Content-Type", "application/x-mpegurl") return response elif action == "delete": request.setResponseCode(http.OK) return "TODO: DELETE FILE: %s" % (filename) elif action == "download": request.setHeader( "Content-Disposition", "attachment;filename=\"%s\"" % (filename.split('/')[-1])) rfile = static.File(six.ensure_binary(filename), defaultType="application/octet-stream") return rfile.render(request) else: return "wrong action parameter" path = getUrlArg(request, "dir") if path != None: pattern = '*' nofiles = False pattern = getUrlArg(request, "pattern", "*") nofiles = getUrlArg(request, "nofiles") != None directories = [] files = [] request.setHeader("content-type", "application/json; charset=utf-8") if fileExists(path): if path == '/': path = '' try: files = glob.glob(path + '/' + pattern) except: # nosec # noqa: E722 files = [] files.sort() tmpfiles = files[:] for x in tmpfiles: if os.path.isdir(x): directories.append(x + '/') files.remove(x) if nofiles: files = [] return json.dumps( { "result": True, "dirs": directories, "files": files }, indent=2) else: return json.dumps( { "result": False, "message": "path %s not exits" % (path) }, indent=2)
def is_psv(buf): '''If the buffer is a PSV file then return True.''' buf_rows = six.BytesIO(six.ensure_binary(buf)) table_set = messytables.CSVTableSet(buf_rows, delimiter='|') return _is_spreadsheet(table_set, 'PSV')
def email_hash(self): e = '' if self.email: e = self.email.strip().lower().encode('utf8') return md5(six.ensure_binary(e)).hexdigest()
def _Utf8ToBase64(s): """Encode a utf-8 string as a base 64 string.""" return six.ensure_text(base64.b64encode(six.ensure_binary(s)))
def CreateObject(self, bucket_uri=None, object_name=None, contents=None, prefer_json_api=False, encryption_key=None, mode=None, mtime=None, uid=None, gid=None, storage_class=None, gs_idempotent_generation=0, kms_key_name=None): """Creates a test object. Args: bucket_uri: The URI of the bucket to place the object in. If not specified, a new temporary bucket is created. object_name: The name to use for the object. If not specified, a temporary test object name is constructed. contents: The contents to write to the object. If not specified, the key is not written to, which means that it isn't actually created yet on the server. prefer_json_api: If true, use the JSON creation functions where possible. encryption_key: AES256 encryption key to use when creating the object, if any. mode: The POSIX mode for the object. Must be a base-8 3-digit integer represented as a string. mtime: The modification time of the file in POSIX time (seconds since UTC 1970-01-01). If not specified, this defaults to the current system time. uid: A POSIX user ID. gid: A POSIX group ID. storage_class: String representing the storage class to use for the object. gs_idempotent_generation: For use when overwriting an object for which you know the previously uploaded generation. Create GCS object idempotently by supplying this generation number as a precondition and assuming the current object is correct on precondition failure. Defaults to 0 (new object); to disable, set to None. kms_key_name: Fully-qualified name of the KMS key that should be used to encrypt the object. Note that this is currently only valid for 'gs' objects. Returns: A StorageUri for the created object. """ bucket_uri = bucket_uri or self.CreateBucket() # checking for valid types - None or unicode/binary text if contents is not None: if not isinstance(contents, (six.binary_type, six.text_type)): raise TypeError('contents must be either none or bytes, not {}'.format( type(contents))) contents = six.ensure_binary(contents) if (contents and bucket_uri.scheme == 'gs' and (prefer_json_api or encryption_key or kms_key_name)): object_name = object_name or self.MakeTempName('obj') json_object = self.CreateObjectJson( contents=contents, bucket_name=bucket_uri.bucket_name, object_name=object_name, encryption_key=encryption_key, mtime=mtime, storage_class=storage_class, gs_idempotent_generation=gs_idempotent_generation, kms_key_name=kms_key_name) object_uri = bucket_uri.clone_replace_name(object_name) # pylint: disable=protected-access # Need to update the StorageUri with the correct values while # avoiding creating a versioned string. md5 = (Base64ToHexHash(json_object.md5Hash), json_object.md5Hash.strip('\n"\'')) object_uri._update_from_values(None, json_object.generation, True, md5=md5) # pylint: enable=protected-access return object_uri bucket_uri = bucket_uri or self.CreateBucket() object_name = object_name or self.MakeTempName('obj') key_uri = bucket_uri.clone_replace_name(object_name) if contents is not None: if bucket_uri.scheme == 'gs' and gs_idempotent_generation is not None: try: key_uri.set_contents_from_string(contents, headers={ 'x-goog-if-generation-match': str(gs_idempotent_generation) }) except StorageResponseError as e: if e.status == 412: pass else: raise else: key_uri.set_contents_from_string(contents) custom_metadata_present = (mode is not None or mtime is not None or uid is not None or gid is not None) if custom_metadata_present: self.SetPOSIXMetadata(bucket_uri.scheme, bucket_uri.bucket_name, object_name, atime=None, mtime=mtime, uid=uid, gid=gid, mode=mode) return key_uri
def download(url, file, writeProgress=None, contextFactory=None, *args, **kwargs): """Download a remote file and provide current-/total-length. @param file: path to file on filesystem, or file-like object. @param writeProgress: function or list of functions taking two parameters (pos, length) See HTTPDownloader to see what extra args can be passed if remote file is accessible via http or https. Both Backends should offer supportPartial. """ scheme, host, port, path, username, password = _parse(url) if scheme == 'ftp': from .FTPProgressDownloader import FTPProgressDownloader if not (username and password): username = '******' password = '******' client = FTPProgressDownloader( host, port, path, file, username, password, writeProgress, *args, **kwargs ) return client.deferred # We force username and password here as we lack a satisfying input method if username and password: from six import PY3, ensure_binary from base64 import b64encode # twisted will crash if we don't rewrite this ;-) url = scheme + '://' + host + ':' + str(port) + path base64string = "%s:%s" % (username, password) base64string = b64encode(ensure_binary(base64string)) if PY3: base64string.decode() AuthHeaders = {"Authorization": "Basic %s" % base64string} if "headers" in kwargs: kwargs["headers"].update(AuthHeaders) else: kwargs["headers"] = AuthHeaders from .HTTPProgressDownloader import HTTPProgressDownloader from twisted.internet import reactor factory = HTTPProgressDownloader(url, file, writeProgress, *args, **kwargs) if scheme == 'https': from twisted.internet import ssl if contextFactory is None: contextFactory = ssl.ClientContextFactory() reactor.connectSSL(host, port, factory, contextFactory) else: reactor.connectTCP(host, port, factory) return factory.deferred
def _fourcc(self): return six.ensure_binary( struct.pack('!I', self._buffer.read_bits(32)), encoding='ascii' )
def _rewrite(self, original_model: rewriter.ModelDescription, rewritten_model: rewriter.ModelDescription): """Rewrites the provided model. Args: original_model: A `ModelDescription` specifying the original model to be rewritten. rewritten_model: A `ModelDescription` specifying the format and location of the rewritten model. Raises: ValueError: If the model could not be sucessfully rewritten. """ if rewritten_model.model_type not in [ rewriter.ModelType.TFLITE_MODEL, rewriter.ModelType.ANY_MODEL ]: raise ValueError( 'TFLiteConverter can only convert to the TFLite format.') # TODO(dzats): We create a temporary directory with a SavedModel that does # not contain an assets or assets.extra directory. Remove this when the # TFLite converter can convert models having these directories. tmp_model_dir = os.path.join(six.ensure_text(rewritten_model.path), 'tmp-rewrite-' + str(int(time.time()))) if fileio.exists(tmp_model_dir): raise ValueError( 'TFLiteConverter is unable to create a unique path ' 'for the temp rewriting directory.') fileio.makedirs(tmp_model_dir) _create_tflite_compatible_saved_model( six.ensure_text(original_model.path), tmp_model_dir) converter = _create_tflite_converter( saved_model_path=tmp_model_dir, quantization_optimizations=self._quantization_optimizations, quantization_supported_types=self._quantization_supported_types, # TODO(b/175699054): Enable once data API is landed. input_data=self._input_data) tflite_model = converter.convert() output_path = os.path.join(six.ensure_text(rewritten_model.path), self._filename) with fileio.open(six.ensure_text(output_path), 'wb') as f: f.write(six.ensure_binary(tflite_model)) fileio.rmtree(tmp_model_dir) copy_pairs = [] if self._copy_assets: src = os.path.join(six.ensure_text(original_model.path), tf.saved_model.ASSETS_DIRECTORY) dst = os.path.join(six.ensure_text(rewritten_model.path), tf.saved_model.ASSETS_DIRECTORY) if fileio.isdir(src): fileio.mkdir(dst) copy_pairs.append((src, dst)) if self._copy_assets_extra: src = os.path.join(six.ensure_text(original_model.path), EXTRA_ASSETS_DIRECTORY) dst = os.path.join(six.ensure_text(rewritten_model.path), EXTRA_ASSETS_DIRECTORY) if fileio.isdir(src): fileio.mkdir(dst) copy_pairs.append((src, dst)) for src, dst in copy_pairs: io_utils.copy_dir(src, dst)
def _pack_unpack(format_str, *args): """wrapper for struct.pack function that converts unicode format string to 'str'""" binary_format_str = six.ensure_binary(format_str) return f(binary_format_str, *args)
def getPLSContent(self): print("loading PLS of stream ", self.name, self.url) getPage(six.ensure_binary(self.url)).addCallback(self._gotPLSContent).addErrback(self._errorPLSContent)
def __call__(self, r): r.headers[b'Authorization'] = ensure_binary('Bearer {}'.format( self.token)) return r
def sha1(self): return hashlib.sha1(six.ensure_binary(self.data)).hexdigest()
def generate_signed_url(service_account_file, bucket_name, object_name, subresource=None, expiration=604800, http_method='GET', query_parameters=None, headers=None): if expiration > 604800: print('Expiration Time can\'t be longer than 604800 seconds (7 days).') sys.exit(1) escaped_object_name = quote(six.ensure_binary(object_name), safe=b'/~') canonical_uri = '/{}'.format(escaped_object_name) datetime_now = datetime.datetime.utcnow() request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ') datestamp = datetime_now.strftime('%Y%m%d') google_credentials = service_account.Credentials.from_service_account_file( service_account_file) client_email = google_credentials.service_account_email credential_scope = '{}/auto/storage/goog4_request'.format(datestamp) credential = '{}/{}'.format(client_email, credential_scope) if headers is None: headers = dict() host = '{}.storage.googleapis.com'.format(bucket_name) headers['host'] = host canonical_headers = '' ordered_headers = collections.OrderedDict(sorted(headers.items())) for k, v in ordered_headers.items(): lower_k = str(k).lower() strip_v = str(v).lower() canonical_headers += '{}:{}\n'.format(lower_k, strip_v) signed_headers = '' for k, _ in ordered_headers.items(): lower_k = str(k).lower() signed_headers += '{};'.format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' if query_parameters is None: query_parameters = dict() query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256' query_parameters['X-Goog-Credential'] = credential query_parameters['X-Goog-Date'] = request_timestamp query_parameters['X-Goog-Expires'] = expiration query_parameters['X-Goog-SignedHeaders'] = signed_headers if subresource: query_parameters[subresource] = '' canonical_query_string = '' ordered_query_parameters = collections.OrderedDict( sorted(query_parameters.items())) for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) canonical_query_string = canonical_query_string[:-1] # remove trailing '&' canonical_request = '\n'.join([ http_method, canonical_uri, canonical_query_string, canonical_headers, signed_headers, 'UNSIGNED-PAYLOAD' ]) canonical_request_hash = hashlib.sha256( canonical_request.encode()).hexdigest() string_to_sign = '\n'.join([ 'GOOG4-RSA-SHA256', request_timestamp, credential_scope, canonical_request_hash ]) # signer.sign() signs using RSA-SHA256 with PKCS1v15 padding signature = binascii.hexlify( google_credentials.signer.sign(string_to_sign)).decode() # scheme_and_host = '{}://{}'.format('https', host) # signed_url = '{}{}?{}&x-goog-signature={}'.format( # canonical_uri, canonical_query_string, signature) signed_url = '{}?{}&x-goog-signature={}'.format(canonical_uri, canonical_query_string, signature) return signed_url
def CreateTempFile(self, tmpdir=None, contents=None, file_name=None, mtime=None, mode=NA_MODE, uid=NA_ID, gid=NA_ID): """Creates a temporary file on disk. Note: if mode, uid, or gid are present, they must be validated by ValidateFilePermissionAccess and ValidatePOSIXMode before calling this function. Args: tmpdir: The temporary directory to place the file in. If not specified, a new temporary directory is created. contents: The contents to write to the file. If not specified, a test string is constructed and written to the file. Since the file is opened 'wb', the contents must be bytes. file_name: The name to use for the file. If not specified, a temporary test file name is constructed. This can also be a tuple, where ('dir', 'foo') means to create a file named 'foo' inside a subdirectory named 'dir'. mtime: The modification time of the file in POSIX time (seconds since UTC 1970-01-01). If not specified, this defaults to the current system time. mode: The POSIX mode for the file. Must be a base-8 3-digit integer represented as a string. uid: A POSIX user ID. gid: A POSIX group ID. Returns: The path to the new temporary file. """ tmpdir = six.ensure_str(tmpdir or self.CreateTempDir()) file_name = file_name or self.MakeTempName(str('file')) if isinstance(file_name, (six.text_type, six.binary_type)): fpath = os.path.join(tmpdir, six.ensure_str(file_name)) else: file_name = map(six.ensure_str, file_name) fpath = os.path.join(tmpdir, *file_name) if not os.path.isdir(os.path.dirname(fpath)): os.makedirs(os.path.dirname(fpath)) if isinstance(fpath, six.binary_type): fpath = fpath.decode(UTF8) with open(fpath, 'wb') as f: contents = (contents if contents is not None else self.MakeTempName( str('contents'))) if isinstance(contents, bytearray): contents = bytes(contents) else: contents = six.ensure_binary(contents) f.write(contents) if mtime is not None: # Set the atime and mtime to be the same. os.utime(fpath, (mtime, mtime)) if uid != NA_ID or int(gid) != NA_ID: os.chown(fpath, uid, int(gid)) if int(mode) != NA_MODE: os.chmod(fpath, int(mode, 8)) return fpath
def generate_ansible_command(self): """ Given that the ``RunnerConfig`` preparation methods have been run to gather the inputs this method will generate the ``ansible`` or ``ansible-playbook`` command that will be used by the :py:class:`ansible_runner.runner.Runner` object to start the process """ if self.binary is not None: base_command = self.binary self.execution_mode = ExecutionMode.RAW elif self.module is not None: base_command = 'ansible' self.execution_mode = ExecutionMode.ANSIBLE else: base_command = 'ansible-playbook' self.execution_mode = ExecutionMode.ANSIBLE_PLAYBOOK exec_list = [base_command] try: if self.cmdline_args: cmdline_args = self.cmdline_args else: cmdline_args = self.loader.load_file('env/cmdline', string_types, encoding=None) if six.PY2: cmdline_args = ensure_binary(cmdline_args, encoding='utf-8') args = shlex.split(cmdline_args) exec_list.extend(args) except ConfigurationError: pass if self.inventory is None: pass elif isinstance(self.inventory, list): for i in self.inventory: exec_list.append("-i") exec_list.append(i) else: exec_list.append("-i") exec_list.append(self.inventory) if self.limit is not None: exec_list.append("--limit") exec_list.append(self.limit) if self.loader.isfile('env/extravars'): exec_list.extend(['-e', '@{}'.format(self.loader.abspath('env/extravars'))]) if self.extra_vars: if isinstance(self.extra_vars, dict) and self.extra_vars: exec_list.extend( [ '-e', '%s' % ' '.join( ["{}=\"{}\"".format(k, self.extra_vars[k]) for k in self.extra_vars] ) ] ) elif self.loader.isfile(self.extra_vars): exec_list.extend(['-e', '@{}'.format(self.loader.abspath(self.extra_vars))]) if self.verbosity: v = 'v' * self.verbosity exec_list.append('-{}'.format(v)) if self.tags: exec_list.extend(['--tags', '{}'.format(self.tags)]) if self.skip_tags: exec_list.extend(['--skip-tags', '{}'.format(self.skip_tags)]) if self.forks: exec_list.extend(['--forks', '{}'.format(self.forks)]) # Other parameters if self.execution_mode == ExecutionMode.ANSIBLE_PLAYBOOK: exec_list.append(self.playbook) elif self.execution_mode == ExecutionMode.ANSIBLE: exec_list.append("-m") exec_list.append(self.module) if self.module_args is not None: exec_list.append("-a") exec_list.append(self.module_args) if self.host_pattern is not None: exec_list.append(self.host_pattern) return exec_list
def post_ratelimited(protocol, session, url, headers, data, allow_redirects=False, stream=False, timeout=None): """ There are two error-handling policies implemented here: a fail-fast policy intended for stand-alone scripts which fails on all responses except HTTP 200. The other policy is intended for long-running tasks that need to respect rate-limiting errors from the server and paper over outages of up to 1 hour. Wrap POST requests in a try-catch loop with a lot of error handling logic and some basic rate-limiting. If a request fails, and some conditions are met, the loop waits in increasing intervals, up to 1 hour, before trying again. The reason for this is that servers often malfunction for short periods of time, either because of ongoing data migrations or other maintenance tasks, misconfigurations or heavy load, or because the connecting user has hit a throttling policy limit. If the loop exited early, consumers of exchangelib that don't implement their own rate-limiting code could quickly swamp such a server with new requests. That would only make things worse. Instead, it's better if the request loop waits patiently until the server is functioning again. If the connecting user has hit a throttling policy, then the server will start to malfunction in many interesting ways, but never actually tell the user what is happening. There is no way to distinguish this situation from other malfunctions. The only cure is to stop making requests. The contract on sessions here is to return the session that ends up being used, or retiring the session if we intend to raise an exception. We give up on max_wait timeout, not number of retries. An additional resource on handling throttling policies and client back off strategies: https://msdn.microsoft.com/en-us/library/office/jj945066(v=exchg.150).aspx#bk_ThrottlingBatch """ thread_id = get_ident() wait = 10 # seconds retry = 0 redirects = 0 # In Python 2, we want this to be a 'str' object so logging doesn't break (all formatting arguments are 'str'). # We activated 'unicode_literals' at the top of this file, so it would be a 'unicode' object unless we convert # to 'str' explicitly. This is a no-op for Python 3. log_msg = str('''\ Retry: %(retry)s Waited: %(wait)s Timeout: %(timeout)s Session: %(session_id)s Thread: %(thread_id)s Auth type: %(auth)s URL: %(url)s HTTP adapter: %(adapter)s Allow redirects: %(allow_redirects)s Response time: %(response_time)s Status code: %(status_code)s Request headers: %(request_headers)s Response headers: %(response_headers)s Request data: %(xml_request)s Response data: %(xml_response)s ''') log_vals = dict( retry=retry, wait=wait, timeout=protocol.TIMEOUT, session_id=session.session_id, thread_id=thread_id, auth=session.auth, url=url, adapter=session.get_adapter(url), allow_redirects=allow_redirects, response_time=None, status_code=None, request_headers=headers, response_headers=None, xml_request=data, xml_response=None, ) try: while True: _back_off_if_needed(protocol.credentials.back_off_until) log.debug( 'Session %s thread %s: retry %s timeout %s POST\'ing to %s after %ss wait', session.session_id, thread_id, retry, protocol.TIMEOUT, url, wait) d_start = time_func() # Always create a dummy response for logging purposes, in case we fail in the following r = DummyResponse(url=url, headers={}, request_headers=headers) try: data = ensure_binary(data) except UnicodeDecodeError: try: data = data.decode('utf-8').encode('utf-8') except UnicodeDecodeError: import chardet encoding_info = chardet.detect(data) data = data.decode('utf-8').encode( encoding_info['encoding']) try: r = session.post(url=url, headers=headers, data=data, allow_redirects=False, timeout=(timeout or protocol.TIMEOUT), stream=stream) except CONNECTION_ERRORS as e: log.debug( 'Session %s thread %s: connection error POST\'ing to %s', session.session_id, thread_id, url) r = DummyResponse(url=url, headers={'TimeoutException': e}, request_headers=headers) finally: log_vals.update( retry=retry, wait=wait, timeout=(timeout or protocol.TIMEOUT), session_id=session.session_id, url=str(r.url), response_time=time_func() - d_start, status_code=r.status_code, request_headers=r.request.headers, response_headers=None if stream else r.headers, xml_request=data, xml_response=None if stream else r.content, ) log.debug(log_msg, log_vals) if _may_retry_on_error(r, protocol, wait): log.info( "Session %s thread %s: Connection error on URL %s (code %s). Cool down %s secs", session.session_id, thread_id, r.url, r.status_code, wait) time.sleep(wait) # Increase delay for every retry retry += 1 wait *= 2 session = protocol.renew_session(session) continue if r.status_code in (301, 302): url, redirects = _redirect_or_fail(r, redirects, allow_redirects) continue break except (RateLimitError, RedirectError) as e: log.warning(e.value) protocol.retire_session(session) raise except Exception as e: # Let higher layers handle this. Add full context for better debugging. log.error(str('%s: %s\n%s'), e.__class__.__name__, str(e), log_msg % log_vals) protocol.retire_session(session) raise if r.status_code == 500 and r.content and is_xml(r.content): # Some genius at Microsoft thinks it's OK to send a valid SOAP response as an HTTP 500 log.debug('Got status code %s but trying to parse content anyway', r.status_code) elif r.status_code != 200: protocol.retire_session(session) _raise_response_errors(r, protocol, log_msg, log_vals) # Always raises an exception log.debug('Session %s thread %s: Useful response from %s', session.session_id, thread_id, url) return r, session
def test_errors_and_encoding(self): six.ensure_binary(self.UNICODE_EMOJI, encoding='latin-1', errors='ignore') with py.test.raises(UnicodeEncodeError): six.ensure_binary(self.UNICODE_EMOJI, encoding='latin-1', errors='strict')