from abc import abstractmethod from pathlib import PurePath WORK_PATH = str(PurePath(__file__).parent.parent.parent) __depends__ = { "district_path": f"{WORK_PATH}/sources/district_type2.txt", } class Feature: def __init__(self): pass @abstractmethod def create(self, text, mode="char"): pass class BasicFeature(Feature): @staticmethod def __create_mode_char(text): def get_num_type(c): chn_num = {'一', '二', '三', '四', '五', '六', '七', '八', '九', '十'} chn_traditional_num = {'甲', '乙', '丙', '丁', '戊', '己', '庚', '辛', '壬', '葵'} if c.isdigit(): return 'digit' if c in chn_num: return 'chn_digit' if c in chn_traditional_num: return 'traditional_digit'
def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0): """ Runs the specified command in the system shell. Returns ======= A tuple of the command output (list of lines) and the command's return code. Arguments ========= cmd: list of tokens that form the command to be run log_filename: the log fiel name for the command's output. Default: derived from command temp_dir: The directory to run the command in. Default: None (uses object default). expected_return_code: The expected return code from the command. If the actula return code does not match, will generate an exception. Default: 0 indent_depth: How deep to indent the tool output in verbose mode. Default: 0 """ # Save the original command orig_cmd = cmd temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir # If no log file is specified the name is based on the executed command log_filename = PurePath( orig_cmd[0] ).name + ".out" if log_filename is None else log_filename # Limit memory usage? memory_limit = [ "ulimit", "-Sv", "{val};".format(val=self._max_memory_mb) ] cmd = memory_limit + cmd if self._max_memory_mb and check_cmd( memory_limit[0]) else cmd # Enable memory tracking? memory_tracking = ["/usr/bin/env", "time", "-v"] cmd = ((memory_tracking + [ "valgrind", "--leak-check=full", "--suppressions=" + str(paths.valgrind_supp), "--error-exitcode=1", "--errors-for-leak-kinds=none", "--track-origins=yes", "--log-file=valgrind.log", "--error-limit=no", ] + cmd if self._valgrind else memory_tracking + cmd) if self._track_memory and check_cmd(memory_tracking[0]) else cmd) # Flush before calling subprocess to ensure output is ordered # correctly if stdout is buffered sys.stdout.flush() # Echo the command? if self._echo_cmd: print(cmd) # Begin timing start_time = time.time() cmd_output = [] cmd_returncode = None proc = None try: # Call the command stderr = None if self._valgrind else subprocess.STDOUT proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, # We grab stdout stderr=stderr, # stderr redirected to stderr universal_newlines=True, # Lines always end in \n cwd=str(temp_dir), # Where to run the command ) # Read the output line-by-line and log it # to a file. # # We do this rather than use proc.communicate() # to get interactive output with (temp_dir / log_filename).open("w") as log_f: # Print the command at the top of the log log_f.write(" ".join(cmd)) log_f.write("\n") # Read from subprocess output for line in proc.stdout: # Send to log file log_f.write(line) # Save the output cmd_output.append(line) # Abort if over time limit elapsed_time = time.time() - start_time if self._timeout_sec and elapsed_time > self._timeout_sec: proc.terminate() # Should now be finished (since we stopped reading from proc.stdout), # sets the return code proc.wait() finally: # Clean-up if we did not exit cleanly if proc: if proc.returncode is None: # Still running, stop it proc.terminate() cmd_returncode = proc.returncode cmd_errored = cmd_returncode != expected_return_code # Send to stdout if self._show_failures and self._verbose: for line in cmd_output: print(indent_depth * self._indent + line, end="") if self._show_failures and cmd_errored and not self._expect_fail: print("\nFailed log file follows({}):".format( str((temp_dir / log_filename).resolve()))) for line in cmd_output: print(indent_depth * self._indent + "<" + line, end="") if cmd_errored: raise CommandError( "Executable {} failed".format(PurePath(orig_cmd[0]).name), cmd=cmd, log=str(temp_dir / log_filename), returncode=cmd_returncode, ) return cmd_output, cmd_returncode
def job_name(self): """ return the name of the job """ return str( PurePath(self.arch()) / self.circuit() / self.script_params())
prepared_X_strat_train, y_strat_train_df, prepared_X_strat_val, y_strat_val_df = \ stratified_shuffle_data_split(prepared_X_train_values, train_labels_df) # grid search setup on XGBClassifier cat_clf = CatBoostClassifier() cat_params = {'iterations': [2e3, 3e3], 'learning_rate': [0.4, 0.6, 0.8]} cat_grid_search = GridSearchCV(cat_clf, cat_params, cv=3, scoring='f1_micro', return_train_score=True, n_jobs=-1, verbose=100) # grid search computation cat_joblib_file = PurePath.joinpath(model_dir, 'cat_grid_search.sav') cat_grid_search_output = grid_search_func( prepared_X_strat_train, y_strat_train_df, # type: ignore cat_grid_search, cat_joblib_file) # type: ignore # output list of RSME in decreasing order grid_results(cat_grid_search_output, prod([len(i) for i in list(cat_params.values())])) # accuracy performance metric y_pred = cat_grid_search_output.predict(prepared_X_strat_val) # type: ignore best_fit_acc_score = accuracy_score(y_strat_val_df, y_pred) # type: ignore print(f'Accuracy for the best-fit model: {best_fit_acc_score:.8f}')
def __init__(self, filepath: str) -> None: protocol, path = get_protocol_and_path(filepath=filepath) self._protocol = protocol self._filepath = PurePath(path) self._filesystem = fsspec.filesystem(protocol=protocol)
def extract_features_labels(): """ This funtion extracts the landmarks features for all images in the folder 'dataset/celeba'. It also extracts the gender label for each image. :return: landmark_features: an array containing 68 landmark points for each image in which a face was detected gender_labels: an array containing the gender label (male=0 and female=1) for each image in which a face was detected """ #print(images_dir) image_paths = [os.path.join(images_dir, l) for l in os.listdir(images_dir)] target_size = None # name = ['eye_color', 'face_shape'] #data = pd.read_csv('./dataset_AMLS_20-21/celeba/labels.csv',sep='\t',header=0,names = name) data = pd.read_csv(os.path.join(basedir, labels_filename),sep='\t') # Y_A10=data['gender'] # Y_A11=data['smiling'] # y_1 = Y_A10 # y_2 = Y_A11 eye = data['eye_color'] faceshape = data['face_shape'] #print(gender_labels[0:5]) # labels_file = open(os.path.join(basedir, labels_filename), 'r') # lines = labels_file.readlines() # # from list get str get array symbols # for i in range(len(rows)): # str0 = listToString(rows[i]) # y = str0.split('\t') # y = np.array(y) # Y_A1[i,0] = y[2] # Y_A1[i,1] = y[3] # #str = str+str0 #gender_labels = {listToString(line).split('\t')[0] : int(listToString(line).split('\t')[3]) for line in lines[1:]} #gender_labels = {listToString(line).split('\t')[3] for line in lines[1:]} #test # for line in lines[1:10]: # #print(line) # gender_labels.append(listToString(line).split('\t')[2]) # a = listToString(line).split('\t') # print(np.array(a)[2]) # #gender_labels = np.array(gender_labels) if os.path.isdir(images_dir): all_features = [] all_labels = [] for img_path in image_paths: #print(img_path) file_name= PurePath(img_path.split('.')[1]).parts[-1] file_name=int(file_name) #print(file_name) # load image img = image.img_to_array( image.load_img(img_path, target_size=target_size, interpolation='bicubic')) #print(img) features, _ = run_dlib_shape(img) #print(features) if features is not None: all_features.append(features) #print(gender_labels[file_name]) all_labels.append(eye[file_name]) #all_labels.append(gender_labels[file_name]) #print(all_labels) landmark_features = np.array(all_features) # labels = (np.array(all_labels, dtype=float) + 1)/2 # simply converts the -1 into 0, so male=0 and female=1 labels = np.array(all_labels, dtype=float) return landmark_features, labels
type=str, help= 'Path to the repo, if not set make sure to run the script from [repo]/utils/web_converter_python/ directory' ) print("\nwebConverter for the deauther2.0 by @xdavidhu\n") args = parser.parse_args() if args.repopath != None: parent = args.repopath print("[+] Using manual path '" + args.repopath + "'\n") else: p = Path.cwd() parent = p.parent.parent license_file_path = str(os.path.join(str(parent), "LICENSE")) q = PurePath('esp8266_deauther') arduino_file_path = str(os.path.join(str(parent / q), "webfiles.h")) datadir = parent / q q = PurePath('web_interface') dir = parent / q q = PurePath('data') datadir = datadir / q if not os.path.exists(str(datadir)): os.mkdir(str(datadir)) q = PurePath('web') compressed = datadir / q if not os.path.exists(str(compressed)): os.mkdir(str(compressed)) html_files = [] css_files = []
def readdir(self): if not self.is_dir: raise NotADirectoryError for path in Path(self.uri).iterdir(): yield FileWrapLocal( PurePath(self.hostname).joinpath(path).as_posix())
def run_test_rule( self, *, config: Type[TestConfiguration], targets: List[TargetWithOrigin], debug: bool = False, include_sources: bool = True, ) -> Tuple[int, str]: console = MockConsole(use_colors=False) options = MockOptions(debug=debug, run_coverage=False) interactive_runner = InteractiveRunner(self.scheduler) workspace = Workspace(self.scheduler) union_membership = UnionMembership( {TestConfiguration: OrderedSet([config])}) def mock_coordinator_of_tests( wrapped_config: WrappedTestConfiguration, ) -> AddressAndTestResult: config = wrapped_config.config return AddressAndTestResult( address=config.address, test_result=config.test_result, # type: ignore[attr-defined] ) result: Test = run_rule( run_tests, rule_args=[ console, options, interactive_runner, TargetsWithOrigins(targets), workspace, union_membership, RegisteredTargetTypes.create([MockTarget]), ], mock_gets=[ MockGet( product_type=AddressAndTestResult, subject_type=WrappedTestConfiguration, mock=lambda wrapped_config: mock_coordinator_of_tests( wrapped_config), ), MockGet( product_type=TestDebugRequest, subject_type=TestConfiguration, mock=lambda _: TestDebugRequest(self.make_ipr()), ), MockGet( product_type=HydratedSources, subject_type=HydrateSourcesRequest, mock=lambda _: HydratedSources( Snapshot( directory_digest=EMPTY_DIRECTORY_DIGEST, files=cast(Tuple[str, ...], ("test.hs", ) if include_sources else ()), dirs=(), ), filespec={"globs": []}, ), ), MockGet( product_type=CoverageReport, subject_type=CoverageDataBatch, mock=lambda _: FilesystemCoverageReport( result_digest=EMPTY_DIRECTORY_DIGEST, directory_to_materialize_to=PurePath("mockety/mock"), report_file=None, ), ), ], union_membership=union_membership, ) return result.exit_code, console.stdout.getvalue()
import sys, os, pickle from pathlib import PurePath current_dir = os.path.realpath(__file__) p = PurePath(current_dir) sys.path.append(str(p.parents[7])+'/metalearners/knn_ranking_method/RMSE') from KNN_ranking_k_3_RMSE import KNN_ranking #Load the selected meta-dataset after performing zero-variance threshold with open(str(p.parents[7])+'/analysis/feature_selection/univariate_selection/ANOVA_X_f1_202.pickle', 'rb') as handle: metadataset_feature_selected = pickle.load(handle) #=====================META-FEATURE EXTRACTION================================== with open(str(p.parents[5])+'/actual/sensor_metafeatures_202.pickle', 'rb') as handle: meta_features = pickle.load(handle) #nested_results is a nested dictionary with all the AUC-ROC performances for each dataset and all models with open(str(p.parents[6])+'/nested_results_roc.pickle', 'rb') as handle: nested_results_roc = pickle.load(handle) """ Remove the meta-features which are not in the meta-dataset (i.e. the features which have not been selected in the feature selection process) """ metafeatures_to_be_removed = [] for metafeature in meta_features.keys(): if metafeature in metadataset_feature_selected.columns: pass else: metafeatures_to_be_removed.append(metafeature)
def is_relative_to(origin: AnyPath, *other: AnyPath) -> bool: try: PurePath(origin).relative_to(*other) return True except ValueError: return False
def cont() -> Iterator[str]: for lhs, rhs in zip(PurePath(p1).parts, PurePath(p2).parts): if lhs == rhs: yield lhs else: break
from os import PathLike, altsep, scandir, sep from pathlib import Path, PurePath, PurePosixPath from typing import Iterator, Optional, Union AnyPath = Union[PathLike, str] POSIX_ROOT = PurePosixPath(altsep or sep) ROOT = PurePath(sep) def walk(path: Union[PurePath, str], dirs: bool = False) -> Iterator[Path]: for s in scandir(path): p = Path(s) if s.is_dir(): if dirs: yield p yield from walk(p, dirs=dirs) else: yield p def is_relative_to(origin: AnyPath, *other: AnyPath) -> bool: try: PurePath(origin).relative_to(*other) return True except ValueError: return False def longest_common_path(p1: AnyPath, p2: AnyPath) -> Optional[PurePath]: def cont() -> Iterator[str]:
async def infer_cc_source_dependencies( request: InferCCDependenciesRequest, cc_files_mapping: CCFilesMapping, cc_infer: CCInferSubsystem, ) -> InferredDependencies: if not cc_infer.includes: return InferredDependencies([]) address = request.sources_field.address wrapped_tgt = await Get( WrappedTarget, WrappedTargetRequest(address, description_of_origin="<infallible>")) explicitly_provided_deps, hydrated_sources = await MultiGet( Get(ExplicitlyProvidedDependencies, DependenciesRequest(wrapped_tgt.target[Dependencies])), Get(HydratedSources, HydrateSourcesRequest(request.sources_field)), ) digest_contents = await Get(DigestContents, Digest, hydrated_sources.snapshot.digest) assert len(digest_contents) == 1 file_content = digest_contents[0] file_path = PurePath(file_content.path) includes = parse_includes(file_content.content.decode()) result: OrderedSet[Address] = OrderedSet() for include in includes: # Skip system-path includes. if include.system_paths_only: continue # First try to resolve the include's path against the same directory where the file is. maybe_relative_file_path = file_path.parent.joinpath(include.path) maybe_relative_address = cc_files_mapping.mapping_not_stripped.get( str(maybe_relative_file_path)) if maybe_relative_address: result.add(maybe_relative_address) continue # Otherwise try source roots. if cc_infer.include_from_source_roots: unambiguous = cc_files_mapping.mapping.get(include.path) ambiguous = cc_files_mapping.ambiguous_files.get(include.path) if unambiguous: result.add(unambiguous) elif ambiguous: explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( ambiguous, address, import_reference="file", context=softwrap(f""" The target {address} includes `{include.path}` in the file {file_content.path} """), ) maybe_disambiguated = explicitly_provided_deps.disambiguated( ambiguous) if maybe_disambiguated: result.add(maybe_disambiguated) return InferredDependencies(sorted(result))
def parse_args_and_go(): args = parser.parse_args() runMedford(PurePath(args.file), args.write_json, args.mode, args.error_mode, args.error_sort, args.action)
def create_jobs(args, configs, longest_name=0, longest_arch_circuit=0, after_run=False): """ Create the jobs to be executed depending on the configs. """ jobs = [] for config in configs: for arch, circuit in itertools.product(config.archs, config.circuits): golden_results = load_parse_results( str( PurePath( config.config_dir).joinpath("golden_results.txt"))) abs_arch_filepath = resolve_vtr_source_file( config, arch, config.arch_dir) abs_circuit_filepath = resolve_vtr_source_file( config, circuit, config.circuit_dir) work_dir = str(PurePath(arch).joinpath(circuit)) run_dir = (str( Path(get_latest_run_dir(find_task_dir(config))) / work_dir) if after_run else str( Path(get_next_run_dir(find_task_dir(config))) / work_dir)) # Collect any extra script params from the config file cmd = [abs_circuit_filepath, abs_arch_filepath] if hasattr(args, "show_failures") and args.show_failures: cmd += ["-show_failures"] cmd += config.script_params if config.script_params else [] cmd += config.script_params_common if config.script_params_common else [] cmd += (args.shared_script_params if hasattr(args, "shared_script_params") and args.shared_script_params else []) # Apply any special config based parameters if config.cmos_tech_behavior: cmd += [ "-cmos_tech", resolve_vtr_source_file(config, config.cmos_tech_behavior, "tech"), ] cmd += ([ "--fix_pins", resolve_vtr_source_file(config, config.pad_file) ] if config.pad_file else []) if config.sdc_dir: cmd += [ "-sdc_file", "{}/{}.sdc".format(config.sdc_dir, Path(circuit).stem), ] parse_cmd = None second_parse_cmd = None qor_parse_command = None if config.parse_file: parse_cmd = [ resolve_vtr_source_file( config, config.parse_file, str(PurePath("parse").joinpath("parse_config")), ) ] if config.second_parse_file: second_parse_cmd = [ resolve_vtr_source_file( config, config.second_parse_file, str(PurePath("parse").joinpath("parse_config")), ) ] if config.qor_parse_file: qor_parse_command = [ resolve_vtr_source_file( config, config.qor_parse_file, str(PurePath("parse").joinpath("qor_config")), ) ] # We specify less verbosity to the sub-script # This keeps the amount of output reasonable if hasattr(args, "verbosity") and max(0, args.verbosity - 1): cmd += ["-verbose"] if config.script_params_list_add: for value in config.script_params_list_add: jobs.append( create_job( args, config, circuit, arch, value, cmd, parse_cmd, second_parse_cmd, qor_parse_command, work_dir, run_dir, longest_name, longest_arch_circuit, golden_results, )) else: jobs.append( create_job( args, config, circuit, arch, None, cmd, parse_cmd, second_parse_cmd, qor_parse_command, work_dir, run_dir, longest_name, longest_arch_circuit, golden_results, )) return jobs
def startList(subscription): import uuid from defusedxml import ElementTree as etree from lxml.etree import Element import datetime uid = str(uuid.uuid4()) r = request # client uid request.args.get('clientUid') # name of template request.args.get('name') # description of template request.args.get('description') # startTime of template startTime = request.args.get('startTime') # callsign of submission user request.args.get('callsign') with open( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(f'{uid}.xml'))), 'w+') as file: file.write( str( open( str( PurePath(Path(MainConfig.ExCheckFilePath), Path(f'{subscription}.xml'))), 'r').read())) file.close() xml = etree.parse(MainConfig.ExCheckChecklistFilePath + '/' + uid + '.xml').getroot() starttime = Element('startTime') starttime.text = startTime details = xml.find('checklistDetails') if details.find('startTime') == None: details.append(starttime) else: details.find('startTime').text = startTime uids = details.find('uid') uids.text = uid details.find('description').text = request.args.get('description') details.find('name').text = request.args.get('name') tasks = xml.find('checklistTasks') for task in tasks: taskuid = task.find('uid') taskuid.text = str(uuid.uuid4()) with open( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(uid + '.xml'))), 'w+') as file: y = etree.tostring(xml) file.write(etree.tostring(xml).decode()) file.close() excheckobj = dbController.query_ExCheck( f'ExCheckData.uid == "{subscription}"', verbose=True, )[0] dbController.create_Excheckchecklist( startTime=datetime.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%S.%fZ'), creatorUid=request.args.get('clientUid'), description=request.args.get('description'), callsign=request.args.get('callsign'), name=request.args.get('name'), uid=uid, filename=f'{uid}.xml', template=excheckobj) return str( open( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(uid + '.xml'))), 'r').read()), 200
from pathlib import PurePath import datahub.core.fields import django.contrib.postgres.indexes from django.db import migrations, models import django.db.models.deletion import mptt.fields import uuid from datahub.core.migration_utils import load_yaml_data_in_migration fixtures = [ PurePath(__file__).parent / '0001_initial_countries_with_iso_codes.yaml', PurePath(__file__).parent / '0001_sectors.yaml', PurePath(__file__).parent / '0001_initial_services.yaml', PurePath(__file__).parent / '0001_initial_overseas_region.yaml', PurePath(__file__).parent / '0001_sector_clusters.yaml', ] def load_fixtures(apps, schema_editor): for fixture in fixtures: load_yaml_data_in_migration(apps, fixture) class Migration(migrations.Migration): initial = True dependencies = [
def accesschecklist(checklistid): return str( open( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(checklistid + '.xml'))), 'r').read())
TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = False # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ STATIC_URL = '/static/' MEDIA_ROOT = PurePath.joinpath(BASE_DIR, 'media') MEDIA_URL = '/media/' CRISPY_TEMPLATE_PACK = 'bootstrap4' LOGIN_REDIRECT_URL = reverse_lazy('user_profile_detail') LOGOUT_REDIRECT_URL = reverse_lazy('account_login') LOGIN_URL = reverse_lazy('account_login') EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' EMAIL_HOST = 'smtp.gmail.com' EMAIL_POST = 587 EMAIL_USE_TLS = True EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER') EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD')
def updatetemplate(checklistid, taskid): from flask import request from defusedxml import ElementTree as etree from FreeTAKServer.controllers.SpecificCoTControllers.SendExcheckUpdateController import SendExcheckUpdateController from FreeTAKServer.controllers.XMLCoTController import XMLCoTController from FreeTAKServer.model.FTSModel.Event import Event from FreeTAKServer.model.RawCoT import RawCoT import uuid import hashlib data = request.data xml = etree.parse( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(checklistid + '.xml')))).getroot() updatedTask = etree.fromstring(data) tasks = xml.find('checklistTasks') for task in tasks: uid = task.find('uid') if uid.text == taskid: tasks.replace(task, updatedTask) else: pass with open( str( PurePath(Path(MainConfig.ExCheckChecklistFilePath), Path(checklistid + '.xml'))), 'w+') as file: file.write(etree.tostring(xml).decode()) file.close() # Create Object to send to client object = Event.ExcheckUpdate() object.setuid(str(uuid.uuid4())) object.setversion('2.0') object.detail.mission.settype("CHANGE") object.detail.mission.settool("ExCheck") object.detail.mission.setname(checklistid) object.detail.mission.setauthorUid(request.args.get("clientUid")) object.detail.mission.MissionChanges.MissionChange.creatorUid.setINTAG( request.args.get("clientUid")) object.detail.mission.MissionChanges.MissionChange.missionName.setINTAG( checklistid) object.detail.mission.MissionChanges.MissionChange.type.setINTAG( "ADD_CONTENT") object.detail.mission.MissionChanges.MissionChange.contentResource.filename.setINTAG( taskid + '.xml') object.detail.mission.MissionChanges.MissionChange.contentResource.hash.setINTAG( str( hashlib.sha256( str( open( MainConfig.ExCheckChecklistFilePath + '/' + checklistid + '.xml', 'r')).encode()).hexdigest())) object.detail.mission.MissionChanges.MissionChange.contentResource.keywords.setINTAG( 'Task') object.detail.mission.MissionChanges.MissionChange.contentResource.name.setINTAG( taskid) object.detail.mission.MissionChanges.MissionChange.contentResource.size.setINTAG( str(len(data))) #TODO: change this value object.detail.mission.MissionChanges.MissionChange.contentResource.submitter.setINTAG( 'atak') object.detail.mission.MissionChanges.MissionChange.contentResource.uid.setINTAG( taskid) '''object = etree.fromstring(templateex) object.uid = uuid.uuid4() object.find('detail').find('mission').type= "CHANGE" object.find('detail').find('mission').name = taskid object.find('detail').find('mission').Uid = request.args.get("clientUid") object.find('detail').find('mission').find('MissionChanges').find('MissionChange').find('creatorUid').text = request.args.get("clientUid") object.find('detail').find('mission').find('MissionChanges').find('MissionChange').find('missionName').text = taskid object.find('detail').find('mission').find('MissionChanges').find('MissionChange').find('filename').text = checklistid+'.xml' object.detail.mission.MissionChanges.MissionChange.contentResource.hash.setINTAG(str(hashlib.sha256(str(data).encode()).hexdigest())) object.detail.mission.MissionChanges.MissionChange.contentResource.keywords.setINTAG('Task') object.detail.mission.MissionChanges.MissionChange.contentResource.name.setINTAG(checklistid) object.detail.mission.MissionChanges.MissionChange.contentResource.size.setINTAG(str(len(data))) #TODO: change this value object.detail.mission.MissionChanges.MissionChange.contentResource.submitter.setINTAG('test') object.detail.mission.MissionChanges.MissionChange.contentResource.uid.setINTAG(checklistid)''' rawcot = RawCoT() xml = XMLCoTController().serialize_model_to_CoT(object) rawcot.xmlString = xml PIPE.put(rawcot) #PIPE.send() return '', 200
else: # No image in the article, look for one on the filesystem file_glob = "*/" + str(file_path.name)[:-4] + "*.svg" i = list(main_dir.glob(file_glob)) if len(i) != 0: image = i[0] else: image = "" logging.debug("Setting the image path") # Set the path if image: image = path.normpath(path.join(path.dirname(file_path), image)) article['image'] = str( PurePath("/azure/architecture/" + str(Path(image).resolve().relative_to(doc_directory))). as_posix()) article['imagepath'] = str(Path(image).resolve()) else: article[ 'image'] = "/azure/architecture/_images/reference-architectures.svg" logging.debug("Parsing ms.custom") if article_meta.get('ms.custom'): if isinstance(article_meta['ms.custom'], list): tags = article_meta['ms.custom'] else: tags = list(article_meta['ms.custom'].split(",")) article.setdefault('tags', []).extend(tags)
def contrib_setup_py( name: str, description: str, build_file_aliases: bool = False, global_subsystems: bool = False, register_goals: bool = False, rules: bool = False, target_types: bool = False, additional_classifiers: Optional[List[str]] = None, **kwargs, ) -> PythonArtifact: """Creates the setup_py for a pants contrib plugin artifact. :param name: The name of the package; must start with 'pantsbuild.pants.contrib.'. :param description: A brief description of what the plugin provides. :param additional_classifiers: Any additional trove classifiers that apply to the plugin, see: https://pypi.org/pypi?%3Aaction=list_classifiers :param build_file_aliases: If `True`, register.py:build_file_aliases must be defined and registers the 'build_file_aliases' 'pantsbuild.plugin' entrypoint. :param global_subsystems: If `True`, register.py:global_subsystems must be defined and registers the 'global_subsystems' 'pantsbuild.plugin' entrypoint. :param register_goals: If `True`, register.py:register_goals must be defined and registers the 'register_goals' 'pantsbuild.plugin' entrypoint. :param rules: If `True`, register.py:rules must be defined and registers the 'rules' 'pantsbuild.plugin' entrypoint. :param target_types: If `True`, register.py:target_types must be defined and registers the 'target_types' 'pantsbuild.plugin' entrypoint. :param kwargs: Any additional keyword arguments to be passed to `setuptools.setup <https://pythonhosted.org/setuptools/setuptools.html>`_. :returns: A setup_py suitable for building and publishing Pants components. """ if not name.startswith("pantsbuild.pants.contrib."): raise ValueError( f"Contrib plugin package names must start with 'pantsbuild.pants.contrib.', given {name}" ) setup_py = pants_setup_py( name, description, additional_classifiers=additional_classifiers, namespace_packages=["pants", "pants.contrib"], **kwargs, ) if build_file_aliases or register_goals or global_subsystems or rules or target_types: rel_path = parse_context.rel_path # NB: We don't have proper access to SourceRoot computation here, but # # we happen to know that: # A) All existing contribs have their contrib_setup_py() invocation in a BUILD file # exactly three path segments under the source root (i.e., they all have a source # root of src/<name>src/python/, and are defined in pants/contrib/<name>/BUILD under # that.) # B) We are not adding any new contribs in the future, as this idiom is going away. # # So we can semi-hackily compute the register module using this knowledge. module = (PurePath(rel_path).relative_to( PurePath(rel_path).parent.parent.parent).as_posix().replace( "/", ".")) entry_points = [] if build_file_aliases: entry_points.append( f"build_file_aliases = {module}.register:build_file_aliases" ) if register_goals: entry_points.append( f"register_goals = {module}.register:register_goals") if global_subsystems: entry_points.append( f"global_subsystems = {module}.register:global_subsystems") if rules: entry_points.append(f"rules = {module}.register:rules") if target_types: entry_points.append( f"target_types = {module}.register:target_types") setup_py.setup_py_keywords["entry_points"] = { "pantsbuild.plugin": entry_points } return setup_py
def _get_parent(self): return FileWrapLocal(PurePath(self.uri).parent.as_posix())
def work_dir(self, run_dir): """ return the work directory of the job """ return str(PurePath(run_dir).joinpath(self._work_dir))
def check_proj_consistency(actual, expected): # Check equality of all project fields (projects themselves are # not comparable), with extra semantic consistency checking # for paths. assert actual.name == expected.name assert actual.path == expected.path if actual.topdir is None or expected.topdir is None: assert actual.topdir is None and expected.topdir is None assert actual.abspath is None and expected.abspath is None assert actual.posixpath is None and expected.posixpath is None else: assert actual.topdir and actual.abspath and actual.posixpath assert expected.topdir and expected.abspath and expected.posixpath a_top, e_top = PurePath(actual.topdir), PurePath(expected.topdir) a_abs, e_abs = PurePath(actual.abspath), PurePath(expected.abspath) a_psx, e_psx = PurePath(actual.posixpath), PurePath(expected.posixpath) assert a_top.is_absolute() assert e_top.is_absolute() assert a_abs.is_absolute() assert e_abs.is_absolute() assert a_psx.is_absolute() assert e_psx.is_absolute() assert a_top == e_top assert a_abs == e_abs assert a_psx == e_psx assert (actual.url == expected.url or (WINDOWS and Path(expected.url).is_dir() and (PurePath(actual.url) == PurePath(expected.url)))) assert actual.clone_depth == expected.clone_depth assert actual.revision == expected.revision assert actual.west_commands == expected.west_commands
def load_tasks(): task_csv = cord_support_dir() / 'TaskDefinitions.csv' tasks = pd.read_csv(PurePath(task_csv)) tasks.SeedQuestion = tasks.SeedQuestion.fillna(tasks.Question) return tasks
stratified_shuffle_data_split(prepared_X_train_values, train_labels_df) # classifiers employed for training classifier_dict = { 'xgb_clf': XGBClassifier(n_estimators=500, learning_rate=0.3, colsample_bytree=0.3, subsample=0.3, early_stopping_rounds=50, verbosity=0), 'sgd_clf': SGDClassifier(loss='modified_huber', n_jobs=-1, early_stopping=True), # 'rf_clf': RandomForestClassifier(n_estimators=500, n_jobs=-1), 'cat_clf': CatBoostClassifier(iterations=2e3, allow_writing_files=False, learning_rate=0.3, loss_function='MultiClass', custom_metric=['Accuracy', 'AUC', 'TotalF1'], verbose=100), 'ada_clf': AdaBoostClassifier(n_estimators=100, learning_rate=0.3), } # creates list of named classifier tuples for training clf_list = clf_func(classifier_dict) # runs actual training on classifiers and outputs results to screen run_clf(prepared_X_strat_train, prepared_X_strat_val, y_strat_train_df, y_strat_val_df, clf_list, model_dir) # save predicted results from test data for DrivenData competition model_clf = load(PurePath.joinpath(model_dir, 'ada_clf.sav')) predicted_y_results = model_clf.predict(prepared_test_values) print(f'type(predicted_y_results): {type(predicted_y_results)}') print(f'predicted_y_results.shape: {predicted_y_results.shape}') print(f'predicted_y_results[:10]: {predicted_y_results[:10]}') predicted_y_results_s = DataFrame(predicted_y_results, index=test_values_df.index, columns=['damage_grade']) predicted_y_results_s.to_csv('predicted_results.csv') print('Done!')
vsqxPath = args.filePath noteLimit = args.noteLimit txtPath = args.outputPath #vsqxPath = '/content/AiDee-simplified.vsqx' assert os.path.exists(vsqxPath) extentStr = vsqxPath.split('.')[-1] if extentStr == 'vsqx': path = PurePath(vsqxPath) vsqx = xml.dom.minidom.parse(str(path)) TEMPO = int(vsqx.getElementsByTagName('tempo')[0].childNodes[1].firstChild.data[:-2]) mf = MIDIFile(len(vsqx.getElementsByTagName('vsTrack')), removeDuplicates=False) time = 0 f = open(txtPath,"w") for trackNo, track in enumerate(vsqx.getElementsByTagName('vsTrack')): for i,note in enumerate(track.getElementsByTagName('note')): if i >= noteLimit: f.close() exit() n = getNoteData(note,'n',i,trackNo) dur = getNoteData(note, 'dur',i,trackNo)
import utils from pathlib import PurePath, Path pp = PurePath(Path.cwd()).parts pdir = PurePath(*pp) import numpy as np import pandas as pd from scipy.spatial import distance from scipy.spatial import KDTree from sklearn.model_selection import TimeSeriesSplit import matplotlib matplotlib.use('TkAgg') import matplotlib.pyplot as plt import classifiers import utils class Embedding: def __init__(self, data): self.data = data def _mutual_information(self, x, y, bins=64): """ fn: calc mutual information between two random variables, I = S(x) + S(y) - S(x,y), between two random variables x and y, where S(x) is the Shannon entropy :param x: 1D array, first var :param y: 1D array, second var
def handle_import(self, name, compilation, rule): """ Re-implementation of the core Sass import mechanism, which looks for files using the staticfiles storage and staticfiles finders. """ original_path = PurePath(name) if original_path.suffix: search_exts = [original_path.suffix] else: search_exts = compilation.compiler.dynamic_extensions if original_path.is_absolute(): # Remove the beginning slash search_path = original_path.relative_to('/').parent elif rule.source_file.origin: search_path = rule.source_file.origin if original_path.parent: search_path = search_path / original_path.parent else: search_path = original_path.parent basename = original_path.stem for prefix, suffix in product(('_', ''), search_exts): filename = PurePath(prefix + basename + suffix) full_filename, storage = get_file_and_storage(str(search_path / filename)) if full_filename: with storage.open(full_filename) as f: return SourceFile.from_file(f, origin=search_path, relpath=filename)
def open(self, audio): self.pause() path = PurePath(audio) if path.suffix == ".wav": self._audio = wave.open(audio, "rb") else: self._audio = None raise ValueError("Unsupported format " + path.suffix) self._length = self._audio.getnframes() / self._audio.getframerate() / self._audio.getnchannels() * 1000 self._device.setchannels(self._audio.getnchannels()) self._device.setrate(self._audio.getframerate()) # 8bit is unsigned in wav files if self._audio.getsampwidth() == 1: self._device.setformat(alsaaudio.PCM_FORMAT_U8) # Otherwise we assume signed data, little endian elif self._audio.getsampwidth() == 2: self._device.setformat(alsaaudio.PCM_FORMAT_S16_LE) elif self._audio.getsampwidth() == 3: self._device.setformat(alsaaudio.PCM_FORMAT_S24_LE) elif self._audio.getsampwidth() == 4: self._device.setformat(alsaaudio.PCM_FORMAT_S32_LE) else: raise ValueError('Unsupported format') self._device.setperiodsize(self._CHUNK) try: with open(str(path.with_suffix(".srt"))) as subs: self._subs = SubRip.parse(subs.read()) except Exception: self._subs = None self._emit_sub_changed_new_thread()
def __call__(self, instance, filename): path = PurePath( instance._meta.app_label, instance._meta.model_name, str(uuid.uuid4()) ) return str(path.with_suffix(PurePath(filename).suffix))
def ignores(self, path): ''' Test if this module ignores the file at "path" ''' test_path = PurePath(path) for exp in self.ignore_patterns: if test_path.match(exp): logger.debug('"%s" ignored' % path) return True return False
def compile(self, *paths): compilation = self.make_compilation() for path in paths: path = PurePath(path) if path.is_absolute(): path = path.relative_to('/') filename, storage = get_file_and_storage(str(path)) with storage.open(filename) as f: source = SourceFile.from_file(f, origin=path.parent, relpath=PurePath(path.name)) compilation.add_source(source) return self.call_and_catch_errors(compilation.run)
def write_path_template(self): rootp = self.reg_root ret = PurePath(self._write_path_template) if self._read_path_template is None and rootp not in ret.parents: if not ret.is_absolute(): ret = rootp / ret else: raise ValueError( ('root: {!r} in not consistent with ' 'read_path_template: {!r}').format(rootp, ret)) return _ensure_trailing_slash(str(ret))
def remap_url(src, remap): """ If src is in a subdirectory of any of the keys in `remap`, rewrite the path to point to a local file relative to the corresponding value. """ path = PurePath(src) for url_path, file_path in remap.items(): print(url_path, file_path) try: rel_path = path.relative_to(url_path) return PurePath(file_path).joinpath(rel_path) except ValueError: continue
def _make_zip(file, workdir, dirprefix): with zipfile.ZipFile(file, 'w', compression=zipfile.ZIP_DEFLATED) as zip: for root, dirs, files in os.walk(workdir): relroot = PurePath(root).relative_to(workdir) dirs[:] = [p for p in dirs if str(relroot / p) not in _IGNORED_DIRS] for f in files: abspath = PurePath(root) / f relpath = abspath.relative_to(workdir) if str(relpath) in _IGNORED_FILES: continue # Python 3.5 zipfile does not support pathlib zip.write(str(abspath), str(dirprefix / relpath))
def convert_uxf_to_other_format(filename, convertion_type): ''' Uses the umlet command line to convert an umlet drawing into another format. Example: # to write './_static/n_ergotic_mongol_1.pdf' convert_uxf_to_other_format( './_static/n_ergotic_mongol_1.uxf', 'pdf') ''' # umlet throws X11 java errors from Linux (hours of wasted time) # so I use the windows version instead cmd_string = \ r"cmd.exe /C '{} -action=convert -format={} -filename={}'". \ format(Windows_Path_To_UMLet, convertion_type, filename) p = subprocess.Popen(cmd_string, stdout=subprocess.PIPE, stdin=subprocess.PIPE, shell=True) p.communicate() p.wait() path = PurePath.joinpath( PurePosixPath(os.getcwd()), PurePosixPath(filename)) basename = Path(filename).resolve().stem basepath = str(path.parents[0]) shutil.move( basepath + '/' + basename + ".uxf.{}".format(convertion_type), basepath + '/' + basename + '.{}'.format(convertion_type) )
def change_directory(self, path): absolute_path = Path(self.get_absolute_path(path, True)) if absolute_path.is_dir(): self.cwd = PurePath('/') / absolute_path.relative_to(self.base_path) return True return False
def generate_client_library(self): cur_dir_path = PurePath(os.getcwd()) if not self.args.protodir: client_libraries_base_dir_path = cur_dir_path.joinpath("client_libraries") if not os.path.exists(client_libraries_base_dir_path): os.makedirs(client_libraries_base_dir_path) else: if os.path.isabs(self.args.protodir): client_libraries_base_dir_path = PurePath(self.args.protodir) else: client_libraries_base_dir_path = cur_dir_path.joinpath(self.args.protodir) if not os.path.isdir(client_libraries_base_dir_path): self._error("directory {} does not exist. Please make sure that the specified path exists".format(client_libraries_base_dir_path)) # Create service client libraries path library_language = self.args.language library_org_id = self.args.org_id library_service_id = self.args.service_id library_dir_path = client_libraries_base_dir_path.joinpath(library_language, get_contract_address(self, "Registry"), library_org_id, library_service_id) metadata = self._get_service_metadata_from_registry() model_ipfs_hash = metadata["model_ipfs_hash"] with TemporaryDirectory() as temp_dir: temp_dir_path = PurePath(temp_dir) proto_temp_dir_path = temp_dir_path.joinpath(library_language, library_org_id, library_service_id) safe_extract_proto_from_ipfs(self._get_ipfs_client(), model_ipfs_hash, proto_temp_dir_path) # Compile proto files compile_proto(Path(proto_temp_dir_path), library_dir_path) self._printout('client libraries for service with id "{}" in org with id "{}" generated at {}'.format(library_service_id, library_org_id, library_dir_path))
def get_absolute_path(self, path, with_base=False): path = PurePath(path) try: new_path = None if path.is_absolute(): new_path = self.base_path / path.relative_to('/') else: new_path = self.base_path / self.cwd.relative_to('/') / path if new_path.exists(): new_path = new_path.resolve() except ValueError: new_path = None if new_path: if with_base: return str(new_path) else: return str(PurePath('/') / new_path.relative_to(self.base_path)) return None
def sane_members(members, destination): resolve = lambda path: realpath(normpath(join(destination, path))) destination = PurePath(destination) for member in members: mpath = PurePath(resolve(member.path)) # Check if mpath is under destination if destination not in mpath.parents: raise BadPathError("Bad path to outside destination directory: {}".format(mpath)) elif member.issym() or member.islnk(): # Check link to make sure it resolves under destination lnkpath = PurePath(member.linkpath) if lnkpath.is_absolute() or lnkpath.is_reserved(): raise BadLinkError("Bad link: {}".format(lnkpath)) # resolve the link to an absolute path lnkpath = PurePath(resolve(lnkpath)) if destination not in lnkpath.parents: raise BadLinkError("Bad link to outside destination directory: {}".format(lnkpath)) yield member
def config_file_paths(self): dirs = self.dirs config_fname = '{}-config.py'.format(self.app_import_name) dpaths = [] if appdirs.system != 'win32': dpaths.extend(dirs.site_config_dir.split(':')) dpaths.append('/etc/{}'.format(self.app_import_name)) dpaths.append('/etc') else: system_drive = PurePath(dirs.site_config_dir).drive system_etc_dir = PurePath(system_drive, '/', 'etc') dpaths.extend(( dirs.site_config_dir, system_etc_dir.joinpath(self.app_import_name).__str__(), system_etc_dir.__str__() )) dpaths.append(dirs.user_config_dir) dpaths.append(osp.dirname(self.app_root_path)) fpaths = [osp.join(dpath, config_fname) for dpath in dpaths] return fpaths
def __init__(self, shellname="zsh", user="", host=""): GObject.GObject.__init__(self) self.commands = {} self.shellname = shellname self.user = user self.host = host self.cwd = PurePath("/") self.base_path = Path(os.path.join(USER_PATH, self.host)) if not os.path.isdir(str(self.base_path)): self.base_path.mkdir(parents=True) self.create_initial_state()
def __compress_local_file(local_path: pathlib.PurePath) -> str: """ :Summary: This function compressed a file in Bzip2 from the given path. :Type local_file: `PurePath` :Parameter local_file: A `PurePath` object containing the path to the file or directory to compress. :ReturnType: `str` :Returns: The path of the file compressed. """ tar_file_path = str(local_path.parent/"compressed_file.tar.bz2") if local_path.is_dir(): __print_message("Compressing the directory...") else: __print_message("Compressing the file...") open(tar_file_path, mode="w").close() # Creates a file with tarfile.open(tar_file_path, mode="w|bz2") as tar_file: tar_file.add(str(local_path), arcname="compressed_file") __print_message("OK!", MessageTypes.SUCCESS) return tar_file_path
class ShellManager(GObject.GObject): """ Manages the available commands for the shell """ def __init__(self, shellname="zsh", user="", host=""): GObject.GObject.__init__(self) self.commands = {} self.shellname = shellname self.user = user self.host = host self.cwd = PurePath("/") self.base_path = Path(os.path.join(USER_PATH, self.host)) if not os.path.isdir(str(self.base_path)): self.base_path.mkdir(parents=True) self.create_initial_state() def create_initial_state(self, remove_existing_content=False): if remove_existing_content: shutil.rmtree(str(self.base_path)) self.base_path.mkdir(parents=True) initial_state_file = os.path.join(DATA_PATH, "{}.zip".format(self.host)) if os.path.isfile(initial_state_file): with zipfile.ZipFile(initial_state_file) as z: z.extractall(str(self.base_path)) @property def prompt(self): return "{}@{}{}> ".format(self.user, self.host, str(self.cwd)) def add_command(self, command): self.commands[command.program_name] = command def find_command(self, command_string): parts = command_string.split() if not parts[0] in self.commands: return None else: return self.commands[parts[0]] def get_absolute_path(self, path, with_base=False): path = PurePath(path) try: new_path = None if path.is_absolute(): new_path = self.base_path / path.relative_to('/') else: new_path = self.base_path / self.cwd.relative_to('/') / path if new_path.exists(): new_path = new_path.resolve() except ValueError: new_path = None if new_path: if with_base: return str(new_path) else: return str(PurePath('/') / new_path.relative_to(self.base_path)) return None def change_directory(self, path): absolute_path = Path(self.get_absolute_path(path, True)) if absolute_path.is_dir(): self.cwd = PurePath('/') / absolute_path.relative_to(self.base_path) return True return False def get_files(self, path): path = PurePath(path) try: actual_path = self.base_path / self.cwd.relative_to('/') / path actual_path = actual_path.resolve() actual_path.relative_to(self.base_path) except ValueError: return [] else: return map(str, [ child.relative_to(self.base_path / actual_path) for child in actual_path.glob('*') ])
# クラス構成 # pathlib.PurePath # pathlib.PurePosixPath # pathlib.PureWindowsPath # pathlib.Path # pathlib.PosixPath # pathlib.WindowsPath from pathlib import Path pprint(Path(',')) # 演算子によるパスの結合-------------------------------------------------- from pathlib import PurePath p = PurePath('/hoge/fuga/') pprint(p / 'piyo') # 純粋パスを扱う(PurePath)------------------------------------------------- from pathlib import PurePath p = PurePath('/hoge/fuga/piyo.txt') pprint(p.drive) pprint(p.root) pprint(p.anchor) pprint(list(p.parents)) pprint(p.parent) pprint(p.name) pprint(p.suffix) pprint(p.stem)
def get_existing_resync_file(self, resync_file): # get the existing resync_file as file URI. # return 'file://'+self.config.get_cfg_resync_dir()+'/'+resync_file p = PurePath(self.config.cfg_resync_dir(), resync_file) return p.as_uri()
def relative_to_sds_root(self, file_in_sub_dir: pathlib.PurePath) -> pathlib.PurePath: return file_in_sub_dir.relative_to(self.root_dir)
def intermediate_path(job: dict, render_path: pathlib.PurePath) -> pathlib.PurePath: """Determine the intermediate render output path.""" name = f'{render_path.name}__intermediate-{job["_created"]:%Y-%m-%d_%H%M%S}' return render_path.with_name(name)
def getMirrorPath(self, filename): '''Get path to mirror file given for a filename.''' return PurePath.joinpath(self.mirrorDir, self.getRelativePath(filename.absolute()))
SESSION_ENGINE = 'django.contrib.sessions.backends.cache' # -- time and locale -- TIME_ZONE = 'Asia/Jerusalem' USE_TZ = True LANGUAGE_CODE = 'ru-RU' USE_I18N = USE_L10N = False # -- paths and urls -- OUR_ROOT = PurePath(__file__).parents[2] MEDIA_ROOT = OUR_ROOT.joinpath('media').as_posix() MEDIA_URL = '/media/' STATIC_ROOT = OUR_ROOT.joinpath('_pub').as_posix() STATIC_URL = '/pub/' STATICFILES_DIRS = (OUR_ROOT.joinpath('pub').as_posix(),) TEMPLATE_DIRS = (OUR_ROOT.joinpath('templates').as_posix(),) ROOT_URLCONF = 'rarjpeg.urls'