def _instances(self): """ Return: [ {'app':'app1', 'intf':'intf1', 'instance_id':xxxxxxx, 'endpoints'}, {'app':'app1', 'intf':'intf2', 'instance_id':xxxxxxx}, {'app':'app1', 'intf':'intf3', 'instance_id':xxxxxxx}, ] """ app_dirs = glob.iglob(os.path.join(self._dataflow_path, '*')) app_dirs = (os.basename(x) for x in app_dirs if os.path.isdir(x)) all_instances = [] if self._project == 'bpc': for record in self._datastore.conn['bpc']['instance_id'].find({}): all_instances.append({ 'instance_id':record['instance_id'], 'app':record['app_name'], 'intf':record['intf_name'], } ) for app in app_dirs: intf_dirs = glob.iglob(os.path.join(self._dataflow_path, app, '*')) intf_dirs = (os.basename(x) for x in intf_dirs if os.path.isdir(x)) for intf in intf_dirs: for instance in all_instances: if instance['app'] == app and instance['intf'] == intf: yield instance raise ValueError('Not find matched instance_id:%s, %s' % (app, intf))
def open(self, path, mode='r'): container, subpath = self._splitAzurePath(path) if mode == 'r': _file = self._service.get_file_to_text(container, os.dirname(subpath), os.basename(subpath)) stream = NamedStringIO(_file.content, name=_file.name) elif mode == 'rb': _file = self._service.get_file_to_bytes(container, os.dirname(subpath), os.basename(subpath)) stream = NamedBytesIO(_file.content, name=_file.name) elif mode == 'w': stream = NamedStringIO(name=subpath) elif mode == 'wb': stream = NamedBytesIO(name=subpath) else: raise NotImplementedError(mode + ' is not supported') try: yield stream finally: if mode == 'w': file_write = self._service.create_file_from_text elif mode == 'wb': file_write = self._service.create_file_from_bytes else: return self.makedirs(os.dirname(path)) file_write(container, os.path.dirname(subpath), os.path.basename(subpath), stream.getvalue())
def _instances(self): """ Return: [ {'app':'app1', 'intf':'intf1', 'instance_id':xxxxxxx, 'endpoints'}, {'app':'app1', 'intf':'intf2', 'instance_id':xxxxxxx}, {'app':'app1', 'intf':'intf3', 'instance_id':xxxxxxx}, ] """ app_dirs = glob.iglob(os.path.join(self._dataflow_path, '*')) app_dirs = (os.basename(x) for x in app_dirs if os.path.isdir(x)) all_instances = [] if self._project == 'bpc': for record in self._datastore.conn['bpc']['instance_id'].find({}): all_instances.append({ 'instance_id': record['instance_id'], 'app': record['app_name'], 'intf': record['intf_name'], }) for app in app_dirs: intf_dirs = glob.iglob(os.path.join(self._dataflow_path, app, '*')) intf_dirs = (os.basename(x) for x in intf_dirs if os.path.isdir(x)) for intf in intf_dirs: for instance in all_instances: if instance['app'] == app and instance['intf'] == intf: yield instance raise ValueError('Not find matched instance_id:%s, %s' % (app, intf))
def mix_audio(input_nonvoc, input_voc, output_dir): sys.stderr.write('mix_audio: {} + {}\r\n'.format(os.basename(input_nonvoc), os.basename(input_voc))) nonvoc = AudioSegment.from_file(input_nonvoc) voc = AudioSegment.from_file(input_voc) # mix sound2 with sound1, starting at 5000ms into sound1) output = nonvoc.overlay(voc, position=0) output_filename = str(uuid.uuid4()) # save the result # voc.export(os.path.join(output_dir, '{}_gt.wav'.format(output_filename)), format='wav') out_file = os.path.join(output_dir, '{}.wav'.format(output_filename)) output.export(out_file, format='wav') print('"{}" "{}"'.format(input_voc, out_file))
def get_icon(name, size): """ Return a gtk.gdk.Pixbuf """ found = False if name.startswith('/'): try: # name is an abslolute pathname to a file pixbuf = gtk.gdk.pixbuf_new_from_file(name) pixbuf = pixbuf.scale_simple(size, size, gtk.gdk.INTERP_BILINEAR) found = True except: # print "get_icon: Cannot get pixbuf from file: " + name name = os.basename(name).splitext()[0] if not found: theme = gtk.icon_theme_get_default() try: pixbuf = theme.load_icon(name, size, gtk.ICON_LOOKUP_USE_BUILTIN) except: try: filename = os.path.join('/usr/share/pixmaps', name) try: pixbuf = gtk.gdk.pixbuf_new_from_file(filename) found = True except: # print "get_icon: [%s] not present in theme" % name pixbuf = theme.load_icon('applications-system', size, gtk.ICON_LOOKUP_USE_BUILTIN) except: pixbuf = theme.load_icon(gtk.STOCK_PREFERENCES, 16, 0) # print "get_icon(name, size): icon name: %s pixbuf size: %dx%d" % (name, pixbuf.get_width(), pixbuf.get_height()) if pixbuf.get_width() < 48 or pixbuf.get_height() < 48: pixbuf = pixbuf.scale_simple(size, size, gtk.gdk.INTERP_BILINEAR) if pixbuf.get_width() > 48 or pixbuf.get_height() > 48: pixbuf = pixbuf.scale_simple(size, size, gtk.gdk.INTERP_BILINEAR) return pixbuf
def get_info(index): pres_data = {} file_name = os.path.join(os.basename(__file__), 'presidents.txt') with open(file_name) as pres_in: for line in pres_in: flds = line[:-1].split(":") if int(flds[0]) == index: pres_data["lastname"] = flds[1] pres_data["firstname"] = flds[2] pres_data["birthdate"] = mkdate(flds[3]) pres_data["deathdate"] = mkdate(flds[4]) pres_data["birthplace"] = flds[5] pres_data["birthstate"] = flds[6] pres_data["termstart"] = mkdate(flds[7]) pres_data["termend"] = mkdate(flds[8]) pres_data["party"] = flds[9] break return pres_data
def write_pid(self, program): pid = str(os.getpid()) filename = os.path.join(self.const.Const().get('pid-path', make=True), os.basename(program)+'.pid') if os.path.exists(filename): self.str2file(filename, pid, mode='a') else: self.str2file(filename, pid, mode='w')
def trans_package(self): # Transfer the packages from the filesystem format on DVD # to the datastream format on to disk for i in pkg_list: pkg_src = os.dirname(i) pkg_name = os.basename(i) os.system('pkgtrans -s %s %s %s' % (i, pkg_src, pkg_name))
def main(): # Default Settings title = 'Image Viewer' image = '' action = basename(argv[0]) back = '' forward = '' hidden = '' if dictionary is None: # Nothing To Do pass elif dictionary.had_key('directory'): # Something To Do if not dictionary.has_key('hidden'): # Just Began Viewing if isdir(dictionary['directory']): # Check For Pickle image_pickle = join(dictionary['directory'], 'image.pickle') if exists(image_pickle): # Load Directory Listing dir_data = load(file(image_pickle)) else: # Create Directory Listing dir_data = update_image_pickle(dictionary['directory']) # START HERE image = WHAT_TO_DECIDE_HERE else: # Post Error Message dictionary['directory'] = 'No Such Directory' else: # Find Next Image show_form(title, image, action, back, forward, hidden)
def downloadFile(data): test = re.search( "https\:\/\/.{1,20}\.my3cx\.it\:5001\/management\/Reports\/.{1,30}_[0-9]{4}_[a-zA-Z0-9]{20}.csv", str(data)).group() nameFile = os.basename(test) urllib3.request.urlretrieve(test, nameFile) return os.path.join('./', nameFile)
def post(self): global worker """ embeddingの開始 """ if "name" in request.json: name = request.json["name"] else: name = os.basename(request.json['audio_id']) ### #src_list = BASE_PATH+request.json['audio_id_list'] #if 'event_list' in request.json: # event_list = request.json['event_list'] ### config_path = CONFIG_PATH + name + ".json" with open(config_path, 'w') as outfile: json.dump(request.json, outfile) feature = request.json[ "feature"] if "feature" in request.json else "mel" method = request.json["method"] if "method" in request.json else "umap" result_path = RESULT_PATH cmd = [ "python", "./src/embedding.py", "--config", config_path, "--name", name, "--feature", feature, "--method", method ] log_path = LOG_PATH + name + ".txt" with open(log_path, 'w') as f: print(" ".join(cmd)) p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT) #p = subprocess.Popen(cmd) pid = int(p.pid) worker[pid] = {"process": p, "name": name} res = {"worker_id": int(p.pid), "name": name} return res
def clone_repo(path): tmpdir = tempfile.mkdtemp(prefix='git_clone_' + os.basename(path)) try: res = subprocess.check_output(['git', 'clone', path, tmpdir]) return tmpdir except CalledProcessError, e: clean_checkout(tmpdir) raise
def write_pid(self, program): pid = str(os.getpid()) filename = os.path.join(self.const.Const().get('pid-path', make=True), os.basename(program) + '.pid') if os.path.exists(filename): self.str2file(filename, pid, mode='a') else: self.str2file(filename, pid, mode='w')
def BigExternal(lab_dir, logger): ''' Ensure large files named in the config/bigexternal.txt are present in the lab directory ''' big_list = os.path.join(lab_dir, 'config', 'bigexternal.txt') if not os.path.isfile(big_list): #print('Missing bigexternal.txt from %s' % big_list) return else: logger.debug('BigExternal file found: %s' % big_list) full = os.path.abspath(lab_dir) homedir = os.environ['HOME'] cache_path = os.path.join(homedir, '.local', 'share', 'labtainer', 'big_cache') lab_cache = os.path.join(cache_path, os.path.basename(lab_dir)) if os.path.isfile(big_list): with open(big_list) as fh: for line in fh: line = line.strip() if len(line) > 0 and not line.startswith('#'): from_file, to_file = line.split() to_path = os.path.join(lab_dir, to_file) if not os.path.isfile(to_path): cache_to_path = os.path.join(lab_cache, to_file) try: os.makedirs(os.path.dirname(cache_to_path)) except: pass if not os.path.isfile(cache_to_path): logger.debug('missing %s, get it from %s success' % (to_path, from_file)) cmd = 'curl -L -R --create-dirs -o %s %s' % ( to_path, from_file) logger.debug('cmd: %s' % cmd) ok = os.system(cmd) logger.debug('result: %d' % ok) shutil.copyfile(to_path, cache_to_path) else: try: os.makedirs(os.path.dirname(to_path)) except: pass logger.debug('got %s from cache' % to_path) shutil.copyfile(cache_to_path, to_path) else: size = os.stat(to_path).st_size if size < 50000: if os.basename(to_path) == 'home.tar': print('Remove the file at %s, and run again.' % to_path) else: print('File at %s is supposed to be large.' % to_path) print( 'Try removing the file and running again. Or get the correct %s from %s' % (to_path, from_file)) exit(1)
def load_game(_): """ Offer a selection of files that exist that are loadable """ import ipdb ipdb.set_trace() load_dir = SAVE_DIR loadable_files = [os.basename(f) for f in os.path.walk(load_dir)] index = menu(loadable_files) with shelve.open(loadable_files[index], 'r') as f: encounter_loop(f['state'])
def add_ose_layer(self): """ """ shutil.copytree(self._get_resource_path(__name__, "data/modules"), "%s/" %self.jboss_home) for module in ['org/jgroups', 'org/jboss/as/clustering/common', 'org/jboss/as/clustering/jgroups', 'org/jboss/as/ee']: for jar in glob.glob("%s/modules/system/layers/base/.overlays/*/%s/main/*.jar" %(self.jboss_home, module)): os.symlink(os.basename(jar), jar)
def remove_pid(self, program): pid = str(os.getpid) filename = os.path.join(self.const.Const().get("pid-path"), os.basename(program)+'.pid') pids = self.file2str(filename).strip().split('\n') or [] for p in pids: if p.strip() == pid: pids.pop(pids.index(p)) if pids: self.str2file(filename, '\n'.join(pids)) else: os.remove(filename)
def persistences(): """ Lists the available persistences """ persistenceFolder = getPersistenceFolder() persistencesList = [] for p in glob.glob(persistenceFolder + "*.pickle"): persistencesList.append(os.basename(p).split('.pickle')[0]) return persistencesList
def filepath(name): if os.path.exists(name): return name name = os.basename(name) if os.sep in name else name for confdir in _conf_directories(): path = os.path.join(confdir, name) if os.path.exists(path): return path return None
def remove_pid(self, program): pid = str(os.getpid) filename = os.path.join(self.const.Const().get("pid-path"), os.basename(program) + '.pid') pids = self.file2str(filename).strip().split('\n') or [] for p in pids: if p.strip() == pid: pids.pop(pids.index(p)) if pids: self.str2file(filename, '\n'.join(pids)) else: os.remove(filename)
def load_pdf_metadata(pdf_path: str, current_obj: Mapping[T, T]) -> Mapping[T, T]: """ Load the pdf metadata """ pdf_name = os.basename(pdf_path) df, limit = parse_pdf(pdf_path) df = df.to_dict() limit = list(limit) current_obj['metadata'] = df current_obj['metadata_dimension'] = limit current_obj['pdf_name'] = pdf_name current_obj['event_stream'].append('metadata') return current_obj
def reimpose_neighbour_masks(source_galaxies, source_masks, target, compressed=True): """Direct this function towards a set of neighbour free MEDS files""" if compressed: suffix = ".fz" else: suffix = "" files = glob.glob("%s/DES*.fits%s"%(source, suffix)) print "Will remove neighbour masking in %d MEDS files." for i, f in files: tile = os.basename(f)[:12] print i, tile m = s.meds_wrapper(f)
def writetrainlist(path_root): oriimage_root = path_root + '/oriimage' classify_temp = os.listdir(oriimage_root) strlist = '' for c in classify_temp: labelimage = os.path.join(oriimage_root, c) #原始图片作为类别标签 croot = os.path.join(path_root, os.basename(c)) for d in os.listdir(croot): trainimage = os.path.join(croot, d) strlist += trainimage + ' ' + labelimage + '\n' txtlist = open(path_root + '.txt', 'w') txtlist.write(strlist) txtlist.close()
def get_product_id(specification, version): ''' Get the product id from the specification and version. @param specification: specification name to create product id from @param version: version to used to create the product @return: product id ''' match = TYPE_TAIL_RE.match(specification) if not match or not match.group(1) in ["job-spec", "hysds-io"]: raise Exception("Invalid specification path") ptype = "job" if match.group(1) == "job-spec" else "hysds-io" name = match.group(2) if name == "": name = get_repo(os.basename(specification)) return "{0}-{1}:{2}".format(ptype, name, version)
def post(self): """ separationの開始 """ global worker src_path=BASE_PATH+request.json['audio_id'] tf_path=TF_PATH+request.json['transfer_function'] if "name" in request.json: name= request.json["name"] else: name=os.basename(request.json['audio_id']) log_path=LOG_PATH+name+".txt" result_path=RESULT_PATH lowest_freq = request.json['lowest_freq'] if 'lowest_freq' in request.json else 1000 interval = request.json['interval'] if 'interval' in request.json else 400 localization=request.json['localization'] event_list=localization["event_list"] tl=convert_events2tl(event_list,interval) loc_filename=RESULT_PATH+name+".loc.json" tl_filename=RESULT_PATH+name+".tl.json" with open(loc_filename, 'w') as fp: json.dump(localization, fp) with open(tl_filename, 'w') as fp: json.dump({"interval": interval, "tl": tl}, fp) os.makedirs(RESULT_PATH+name+"_sep/",exist_ok=True) cmd=[ "micarrayx-separate", tf_path, src_path, "--timeline", tl_filename, #"--min_freq", str(lowest_freq), "--out", RESULT_PATH+name+"_sep/sep", "--out_sep_spectrogram_fig", "--out_sep_spectrogram_csv", ] with open(log_path, 'w') as f: print(" ".join(cmd)) p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT) #print(cmd) #p = subprocess.Popen(cmd) pid=int(p.pid) worker[pid]={"process":p,"name":name} res={"worker_id":int(p.pid),"name":name} return res
def _convert_to_trRecord(fileName, buffer4Image, label, text, height, width): """Builds proto""" color = 'RGB' channel = 3 image_formate = 'JPEG' image = tf.train.Example(features=tf.train.Features(feature={ 'image/height': _int64_feature(height), 'image/width': _int64_feature(width), 'image/color': _bytes_feature(tf.compat.as_bytes(color)), 'image/channel': _int64_feature(channel), 'image/class/label': _int64_feature(label), 'image/class/text': _bytes_feature(tf.compat.as_bytes(text)), 'image/format': _bytes_feature(tf.compat.as_bytes(os.basename(fileName))), 'image/encoded': _bytes_feature(tf.compat.as_bytes(buffer4Image))})) return image
def regroup_tarfiles(cc): """Re-group all chunks again per [Case folder] compressed file. First all chunks are copied to the node scratch disc, then start working on them. This only works on a node with PBS stuff. Make sure to maintain the same location as defined by the tags! [res_dir] and [Case folder] could be multiple directories deep, bu the final archive will only contain the files (no directory structure), and the name of the archive is that of the last directory: /[res_dir]/[Case folder]/[Case folder].tar.xz /res/dir/case/folder/dlcname/dlcname.tar.xz Parameters ---------- path_pattern : str /path/to/files/*.tar.xz """ USER = os.getenv('USER') PBS_JOBID = os.getenv('PBS_JOBID') scratch = os.path.join('/scratch', USER, PBS_JOBID) src = os.getenv('PBS_O_WORKDIR') path_pattern = '/home/dave/SimResults/NREL5MW/D0022/prepost-data/*.xz' for ffname in tqdm(glob.glob(path_pattern)): appendix = os.path.basename(ffname).split('_')[0] with tarfile.open(ffname, mode='r:xz') as tar: # create new tar files if necessary for each [Case folder] for tarinfo in tar.getmembers(): t2_name = os.path.basename(os.path.dirname(tarinfo.name)) t2_dir = os.path.join(os.path.dirname(path_pattern), t2_name) if not os.path.isdir(t2_dir): os.makedirs(t2_dir) t2_path = os.path.join(t2_dir, t2_name + '_%s.tar' % appendix) fileobj = tar.extractfile(tarinfo) # change the location of the file in the new archive: # the location of the archive is according to the folder # structure as defined in the tags, remove any subfolders tarinfo.name = os.basename(tarinfo.name) with tarfile.open(t2_path, mode='a') as t2: t2.addfile(tarinfo, fileobj)
def find(self, query): """ Finds nodes based on the query. args: metric query. Returns a list of Nodes. """ cleaned_query = query.replace('\\', '') query_parts = cleaned_query.split('.') for root_dir in self.storage_dirs: for abs_file_path, metric_name in self._find( root_dir, query_parts): # Filter out . and .. if os.basename(abs_file_path).startswith('.'): continue
def m(): excluese_times = get_json('git_commit/timecmt.json') times = generate_timeline(excluese_times) excluese_file = get_json('git_commit/ex_file.json') idx = 0 for file in get_files('/Users/chenxin/tools/algorithms', excluese_file): excluese_file.append(os.basename(file)) time = times[idx] fackaddfile( file, time + ' {}:{}:{}'.format(get_radom(8, 20), get_radom(0, 60), get_radom(0, 60))) excluese_times.append(time) idx += 1 dumps(excluese_times, 'base.json') dumps(excluese_file, 'ex_files.json')
def reimpose_neighbour_masks(source_galaxies, source_masks, target, compressed=True): """Direct this function towards a set of neighbour free MEDS files""" if compressed: suffix = ".fz" else: suffix = "" files = glob.glob("%s/DES*.fits%s" % (source, suffix)) print "Will remove neighbour masking in %d MEDS files." for i, f in files: tile = os.basename(f)[:12] print i, tile m = s.meds_wrapper(f)
def ndPyLibExportCam(isImagePlane, isFbx, isAbc, frameHandle, CameraScale): if cmds.file(q=True, modified=True) == 1: ndPyLibPlatform('please save scene file...') else: filepath = cmds.file(q=True, sceneName=True) filename = os.basename(filepath) project = cmds.match('P:/Project/[a-zA-Z0-9]+/', filepath) roll = cmds.match('/roll[a-zA-Z0-9]+/', filepath) roll = cmds.match('roll[a-zA-Z0-9]+', roll) sequence = cmds.match('/s[0-9]+[a-zA-Z]*/', filepath) sequence = cmds.match('s[0-9]+[a-zA-Z]*/', filepath) shot = cmds.match('/c[0-9]+[a-zA-Z_]*/', filepath) shot = cmds.match('c[0-9]+[a-zA-Z_]*/', shot) shotpath = os.path.join(project, 'shots', roll, sequence, shot) publishpath = os.path.join(shotpath, 'publish', 'Cam', filename) oFilename = os.path.join(sequence, shot, '_cam') ndPyLibPlatform('project : ' + project) ndPyLibPlatform('roll : ' + roll) ndPyLibPlatform('sequence : ' + sequence) ndPyLibPlatform('shot : ' + shot) ndPyLibPlatform('filename : ' + filename) if os.exists(shotpath) != 1: ndPyLibPlatform('no exist folder...') else: if not os.exists(publishpath): os.mkdir(publishpath, md=True) outputfiles = ndPyLibExportCam_exportCamera( publishpath, oFilename, isImagePlane, isFbx, isAbc, frameHandle, CameraScale) for o in outputfiles: ndPyLibPlatform('output file : ' + o) for o in outputfiles: destFile = os.path.dirname( os.path.dirname(o)) + '/' + os.path.basename(o, '') cmds.sysFile(o, copy=destFile)
def _read(self, path): if os.path.exists(path) is False: logging.error('File {0} is not found'.format(path)) raise Exception("File not found") basename = os.basename(path) if basename.find('.csv') != -1: logging.info("Try to load .csv file is loaded") return np.loadtxt(open(path,"rb"),delimiter=",",skiprows=1) else: #load every line f = open(path, 'r') result = [] logging.info("Try to load data from file") lines = f.readlines() f.close() for line in lines: result.append([float(item) for item in line.split()]) return np.array(result)
def main(): them = 'each_wordsquare' me = 'square_meaning' dir_path = os.path.dirname(os.path.realpath(__file__)) download_path = os.path.join(dir_path, "downloaded_images") try: os.makedirs(download_path) except OSError as exc: pass my_tweets = get_tweets(me) their_tweets = get_tweets(them) their_tweets_by_id = {x.id: x for x in their_tweets} previous_tweets = set(x.in_reply_to_status_id for x in my_tweets if x.in_reply_to_status_id is not None) new_tweets = set(their_tweets_by_id.keys()) - previous_tweets for tweet in sorted(new_tweets): text = their_tweets_by_id[tweet].text image_meta = webdriver.load_image_info(text, 5) for image in image_meta: if image.get(SUMMARY): try: image_name = os.path.join( download_path, os.path.basename(image.get(IMAGE))) image_name = image_name.split('?')[0] webdriver.download_image(image.get(IMAGE), image_name) send_tweet(image_name, image.get(SUMMARY), them, tweet) except: continue else: break else: for image in image_meta: if image.get(PAGE_TITLE): image_name = os.path.join(download_path, os.basename(image.get(IMAGE))) webdriver.download_image(image.get(IMAGE), image_name)
def load_page_data(img_dir: str, current_obj: Mapping[T, T]) -> Mapping[T, T]: """ Iterate through the img directory, and retrieve the page level data """ page_data = [] for f in glob.glob(f'{img_tmp}/*'): page_obj = {} page_num = int(os.basename(f)) img = Image.open(f) width, height = img.size with open(f, 'rb') as bimage: bstring = bimage.read() page_obj['bytes'] = bstring page_obj['page_width'] = width page_obj['page_height'] = height page_obj['page_num'] = page_num page_data.append(page_obj) current_obj['page_data'] = page_data current_obj['event_stream'] = ['imagedata'] return current_obj
def regularStylePath(self): if "Italic" not in os.path.basename(self.familyPath): basicStylePath_ = os.path.join( self.familyPath, os.path.basename(self.familyPath).strip() + "-Regular.ufo") if not os.path.exists(basicStylePath_): basicStylePath_ = os.path.join( self.familyPath, os.path.basename(self.familyPath).strip() + "-Light.ufo") if not os.path.exists(basicStylePath_): basicStylePath_ = os.path.join( self.familyPath, os.path.basename(self.familyPath).strip() + "-Thin.ufo") else: basicStylePath_ = os.path.join( self.familyPath, os.basename(self.familyPath).strip() + "-Italic.ufo") print(" >>> " + self.n + "Variable has failed.\n "\ " A static ttf version of the 'Regular' weight"\ "\n will be generated instead.") return basicStylePath_
def get_utkface_np_data(img_dir, img_size): # annotation format: [age]_[gender]_[race]_[date&time].jpg np_imgs = [] labels = [] imgs_list = glob(os.path.join(img_dir, "*.jpg")) for img in imgs_list: pil_img = Image.open(img) resized_pil_img = pil_img.resize((img_size, img_size)) np_img = np.asarray(resized_pil_img) name_split = img.split("_") if not len(name_split) == 4: raise ValueError("annotation format is not correct for {}" .format(os.basename(img))) age, gender, race, _ = name_split np_imgs.append(np_img) labels.append((age, gender, race)) np_imgs = np.array(np_imgs) labels = np.array(labels) return np_imgs, labels
def find_latest_dl_date(save_dir): latest_date = 0 filepaths = glob.glob(save_dir + '/*.pdf') for filepath in filepaths: filename = os.basename(filepath) filedate = int(filename.split('_')[2]) if filedate > latest_date: latest_date = filedate latest_date = str(latest_date) year = int(latest_date[:4]) month = int(latest_date[4:6]) day = int(latest_date[6:]) last_dl_date = datetime.datetime(year,month,day,0,0,0) return last_dl_date
def BigExternal(lab_dir): ''' Ensure large files named in the config/bigexternal.txt are present in the lab directory ''' big_list = os.path.join(lab_dir, 'config', 'bigexternal.txt') if not os.path.isfile(big_list): #print('Missing bigexternal.txt from %s' % big_list) return full = os.path.abspath(lab_dir) if os.path.isfile(big_list): with open(big_list) as fh: for line in fh: line = line.strip() if len(line) > 0 and not line.startswith('#'): from_file, to_file = line.split() to_path = os.path.join(lab_dir, to_file) if not os.path.isfile(to_path): cmd = 'curl -L -R --create-dirs -o %s %s' % (to_path, from_file) print('cmd: %s' % cmd) ok = os.system(cmd) print('missing %s, get it from %s success %d' % (to_path, from_file, ok)) else: size = os.stat(to_path).st_size if size < 50000: if os.basename(to_path) == 'home.tar': print('Remove the file at %s, and run again.' % to_path) else: print('File at %s is supposed to be large.' % to_path) print( 'Try removing the file and running again. Or get the correct %s from %s' % (to_path, from_file)) exit(1)
def save(self): """Save ID3 data.""" try: self.id3.save(v1=2) except: raise Exception ("Except error: ID3 from file %s was NOT saved." %(os.basename(self.f)))
def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup( name='sagemaker-chainer-container', version='1.0', description= 'Open source library template for creating containers to run on Amazon SageMaker.', packages=find_packages(where='src', exclude='test'), package_dir={ 'sagemaker_chainer_container': 'src/sagemaker_chainer_container' }, py_modules=[ os.splitext(os.basename(path))[0] for path in glob('src/*.py') ], long_description=read('README.rst'), author='Amazon Web Services', license='Apache License 2.0', classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', ], # Temporarily freeze sagemaker-containers version to 2.2.5 until we have a proper fix # freeze numpy version because of the python2 bug
from glob import glob import os from setuptools import setup, find_packages def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup(name="sagemaker", version="1.9.2", description="Open source library for training and deploying models on Amazon SageMaker.", packages=find_packages('src'), package_dir={'': 'src'}, py_modules=[os.splitext(os.basename(path))[0] for path in glob('src/*.py')], long_description=read('README.rst'), author="Amazon Web Services", url='https://github.com/aws/sagemaker-python-sdk/', license="Apache License 2.0", keywords="ML Amazon AWS AI Tensorflow MXNet", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", ],
sys.stdout.flush() try: return array(out).transpose() except: return out if __name__ == '__main__': import sys,os fname=sys.argv[1] exargs={} for a in sys.argv[2:]: if a[:5]=='erng=': exargs['erng']=float(a[5:]) if fname[-4:]=='.npy': fdir=os.dirname(fname) fname=os.basename(fname) rep=meas_all(fname,fdir) else: init_single(fname) rep=anal(range(len(dt))) for a in rep: print a global counter counter=0 def bump(w2=310,w1=1500,vl=None,slim=[2.2,2.6],rep=1,min_dis=0.015,loud=0,fit_wid=2): global counter if vl==None: oxide=polyval(spectra.cau_sio2,px) epsil=profit.dielect(px,spectra.lor_si_low[0],spectra.lor_si_low[1])
s = re.search(r"((90[0-9]{7})(\s*\n*\s*)(([A-Z][a-z]*[^\S\r\n]*)*\n*[^\S\r\n]*)(([A-Z][a-z]*[^\S\r\n]*)*)*)", toWrite) print s untrimmed = ''.join([i for i in s.group() if not i.isdigit()]) trimmed1 = untrimmed.strip() trimmed2 = trimmed1.replace("\n", ", ") candidateName = trimmed2.replace(", ,",", ") print "Candidate Name: " + str(candidateName) if #male filename = directory + "/Male/" + candidateName + "/" + candidateName + " - Application.txt" else #female filename = directory + "/Female/" + candidateName + "/" + candidateName + " - Application.txt" # Create the file, writing the GTID and then the application content # We have the write the GTID because it was rmemoved on the split. newFile = file(filename, "w") newFile.write(toWrite) newFile.close() currentpath = os.path.realpath(__file__) thisFile = os.basename(__file__) directorypath = re.split(r"(.*)\\SSSCode\\Modules\\"+thisFile, currentpath) directory = directorypath[1] + "\SSSCodeTestingMaterial\StaffSelectionOutput" sortApps(sampletxt, directory)
for o in (0,1): # yes, it's not quite right to use skyflat_o*r*c skycont_c = (bkg_oyc[o].T[isskycont_oyc[o].T]/ \ skyflat_orc[o].T[isskycont_oyc[o].T]).reshape((cols,-1)).mean(axis=-1) skycont_yc = skycont_c*skyflat_orc[o] profile_oyc[o] -= skycont_yc rblk = 1; cblk = int(cols/16) profile_oyc[o] = blksmooth2d(profile_oyc[o],(okprof_oyc[o] & ~isline_oyc[o]), \ rblk,cblk,0.25,mode='mean') for c in range(cols): psf_orc[o,:,c] = shift(profile_oyc[o,:,c],-drow_oc[o,c],cval=0,order=1) isbkgcont_orc[o,:,c] = shift(isbkgcont_oyc[o,:,c].astype(int),-drow_oc[o,c],cval=0,order=1) > 0.1 isnewbadbin_orc[o,:,c] = shift(isnewbadbin_oyc[o,:,c].astype(int),-drow_oc[o,c],cval=1,order=1) > 0.1 targetrow_od[o,0] = trow_o[o] - np.argmax(isbkgcont_orc[o,trow_o[o]::-1,cols/2] > 0) targetrow_od[o,1] = trow_o[o] + np.argmax(isbkgcont_orc[o,trow_o[o]:,cols/2] > 0) maprow_od = np.vstack((edgerow_od[:,0],targetrow_od[:,0],targetrow_od[:,1],edgerow_od[:,1])).T maprow_od += np.array([-2,-2,2,2]) # pyfits.PrimaryHDU(psf_orc.astype('float32')).writeto('psf_orc.fits',clobber=True) # pyfits.PrimaryHDU(skyflat_orc.astype('float32')).writeto('skyflat_orc.fits',clobber=True) return psf_orc,skyflat_orc,isnewbadbin_orc,isbkgcont_orc,maprow_od,drow_oc #--------------------------------------------------------------------------------------- if __name__=='__main__': infilelist=sys.argv[1:] for file in infilelist: hdulist = pyfits.open(file) name = os.basename(file).split('.')[0] psf_orc,skyflat_orc,isnewbadbin_orc,maprow_od,drow_oc = specpolsignalmap(hdulist) pyfits.PrimaryHDU(psf_orc.astype('float32')).writeto(name+'_psf.fits',clobber=True) pyfits.PrimaryHDU(skyflat_orc.astype('float32')).writeto(name+'_skyflat.fits',clobber=True)
def runGame(userIDs, muValues, sigmaValues): # Setup working path workingPath = "workingPath" if os.path.exists(workingPath): shutil.rmtree(workingPath) os.makedirs(workingPath) os.chmod(workingPath, 0o777) shutil.copyfile("TR_environment_main.py", os.path.join(workingPath, "TR_environment_main.py")) shutil.copyfile("TR_environment_networking.py", os.path.join(workingPath, "TR_environment_networking.py")) sandbox = Sandbox(workingPath) # Unpack and setup bot files botPaths = [os.path.join(workingPath, str(userID)) for userID in userIDs] for botPath in botPaths: os.mkdir(botPath) for a in range(len(userIDs)): unpack("../outputs/TR/"+ str(userIDs[a]) + ".zip", botPaths[a]) for botPath in botPaths: print(botPath) os.chmod(botPath, 0o777) os.chmod(os.path.join(botPath, "run.sh"), 0o777) # Build the shell command that will run the game. Executable called environment houses the game environment runGameShellCommand = "python3 /var/www/nycsl/problems/workers/"+workingPath+"/TR_environment_main.py " for botPath in botPaths: runGameShellCommand += "\"cd "+os.path.abspath(botPath)+"; "+os.path.join(os.path.abspath(botPath), "run.sh")+"\" " print(runGameShellCommand) # Run game sandbox.start(runGameShellCommand) lines = [] while True: line = sandbox.read_line(200) print(line) if line == None: break lines.append(line) # Get player ranks and scores by parsing shellOutput if "won!" in lines[-2]: winnerIndex = int(lines[-2][len("Player ") : -len("won!")]) - 1 loserIndex = (1 if winnerNumber == 2 else 2)-1 else: winnerIndex = random.randrange(0, 2) loserIndex = 0 if winnerIndex == 1 else 1 winnerID = userIDs[winnerIndex] loserID = userIDs[loserIndex] # Update trueskill mu and sigma values winnerRating = trueskill.Rating(mu=float(muValues[winnerIndex]), sigma=float(sigmaValues[winnerIndex])) loserRating = trueskill.Rating(mu=float(muValues[loserIndex]), sigma=float(sigmaValues[loserIndex])) winnerRating, loserRating = trueskill.rate_1vs1(winnerRating, loserRating) cursor.execute("UPDATE Submission SET mu = %f, sigma = %f, score = %d WHERE userID = %d and problemID = %d" % (winnerRating.mu, winnerRating.sigma, int(newRatings[0].mu - (3*newRatings[1].sigma)), winnerID, TRON_PROBLEM_ID)) cursor.execute("UPDATE Submission SET mu = %f, sigma = %f, score = %d WHERE userID = %d and problemID = %d" % (loserRating.mu, loserRating.sigma, int(newRatings[1].mu - (3*newRatings[1].sigma)), loserID, TRON_PROBLEM_ID)) cnx.commit() # Get replay file by parsing shellOutput replayFilename = lines[-1][len("Output file is stored at ") : len(lines[-1])] shutil.move(os.path.join(workingPath, replayFilename), "../storage") # Store results of game cursor.execute("INSERT INTO Game (replayFilename) VALUES (\'"+os.basename(replayFilename)+"\')") cnx.commit() cursor.execute("SELECT gameID FROM Game WHERE replayFilename = \'"+replayFilename+"\'") gameID = cursor.fetchone()['gameID'] cursor.execute("INSERT INTO GameToUser (gameID, userID, rank, index) VALUES (%d, %d, %d)" % (gameID, winnerID, 0, 0 if userIDs[0] == winnerID else 1)) cursor.execute("INSERT INTO GameToUser (gameID, userID, rank, index) VALUES (%d, %d, %d)" % (gameID, loserID, 1, 0 if userIDs[0] == loserID else 1)) cnx.commit() # Delete working path shutil.rmtree(workingPath)
sumaclust_centroids_basename = sumaclust_output_basename + '.centroids' sumaclust_centroids_basepath = os.path.join(args.db_dir, sumaclust_output_basename) sumaclust_centroids_filename = sumaclust_centroids_basename + '.fasta' sumaclust_centroids_filepath = os.path.join(args.db_dir, sumaclust_centroids_filename) clustered_ref_db_basename = cleaned_complete_ref_db_basename + '_NR{0}'.format(clustering_id_threshold_int) if args.by_kingdom: clustered_ref_db_basename += '_bk' clustered_ref_db_basepath = os.path.join(args.db_dir, clustered_ref_db_basename) clustered_ref_db_filename = clustered_ref_db_basename + '.fasta' clustered_ref_db_filepath = os.path.join(args.db_dir, clustered_ref_db_filename) # This is the output MATAM db basepath to pass to matam_assembly.py output_ref_db_basename = clustered_ref_db_basename if args.out_db_name: output_ref_db_basename = os.basename(args.out_db_name) output_ref_db_basepath = os.path.join(args.db_dir, output_ref_db_basename) # This is the output MATAM db file names # For the complete db fasta file output_complete_ref_db_basename = output_ref_db_basename + '.complete' output_complete_ref_db_basepath = os.path.join(args.db_dir, output_complete_ref_db_basename) output_complete_ref_db_filename = output_complete_ref_db_basename + '.fasta' output_complete_ref_db_filepath = os.path.join(args.db_dir, output_complete_ref_db_filename) # For the complete db taxo file output_complete_ref_db_taxo_filename = output_complete_ref_db_basename + '.taxo.tab' output_complete_ref_db_taxo_filepath = os.path.join(args.db_dir, output_complete_ref_db_taxo_filename) # For the clustered db fasta file output_clustered_ref_db_basename = output_ref_db_basename + '.clustered' output_clustered_ref_db_basepath = os.path.join(args.db_dir, output_clustered_ref_db_basename) output_clustered_ref_db_filename = output_clustered_ref_db_basename + '.fasta' output_clustered_ref_db_filepath = os.path.join(args.db_dir, output_clustered_ref_db_filename)
def _main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-r', '--region', choices=config.AMI_ID_BY_REGION.keys(), help="Where in the world to create the VPN (default: %(default)s)", default='eu-west-1') parser.add_argument('-v', '--vpn-type', choices=vpntypes.TYPES.keys(), help="The type of feed to fetch (default: %(default)s)", default='ipsec') parser.add_argument('-a', '--api-key', help="Your AWS API key") parser.add_argument('-s', '--secret-key', action='store_true', help="Prompts for your AWS secret key secret_key interactively (no echo)") parser.add_argument('-S', '--no-prompt-secret', help="Your AWS secret key will be read from the arguments") parser.add_argument('-i', '--identity-file', help="The path to your key pair (not required by all VPN types)") parser.add_argument('-I', '--key-name', help="The name of the keypair to use. ") parser.add_argument('-c', '--config', help="Path to alternate config file", default=os.path.expanduser('~/.ephemvpnrc')) parser.add_argument('-V', '--verbose', action='store_true', help="Be more verbose") parser.add_argument('-q', '--quiet', action='store_true', help="Output nothing. For non-interactive unattended launches, assumes yes to all questions") parser.add_argument('-t', '--running-time', help="The time at or after which the VPN will terminate itself (default: %(default)s)", default='1h') parser.add_argument('-G', '--generate-config', action='store_true', help='Path to create a config file at (recommended {})'.format(os.path.expanduser('~/.ephemvpnrc'))) args = parser.parse_args() log.info(args) if args.generate_config: return _gen_config(args.generate_config) # parse config file cf = _parse_conf(args.config) secret_key = None if args.secret_key: secret_key = getpass.getpass() elif args.no_prompt_secret: secret_key = args.no_prompt_secret if args.api_key: config.AWS_API_KEY = args.api_key if secret_key: config.AWS_SECRET_KEY = secret_key if args.identity_file: config.LOCAL_AWS_KEY_FILE = args.identity_file if args.key_name: config.AWS_KEY_FILE = args.key_name else: name, ext = os.splitext(os.basename(args.identity_file)) config.AWS_KEY_FILE = name elif args.key_name: config.AWS_KEY_FILE = name # sanity check print config.AWS_API_KEY if not config.AWS_API_KEY or not config.AWS_SECRET_KEY or not len(config.AWS_API_KEY) > 0 or not len(config.AWS_SECRET_KEY) > 0: log.info("AWS Credentials required") parser.print_usage() return 1 if not config.AWS_KEY_FILE: log.info("AWS Keypair name required") parser.print_usage() return 1 # calculate running time d_min, shutdown_t = _parse_running_time(args.running_time) # at scheduler doesn't support subminute periods if d_min < 1: log.error("Error:running time must be greater than one minute") return 1 log.info("ephemvpn v{}".format(ephemvpn.__version__)) log.info("summoning one {} vpn".format(args.vpn_type)) if not args.quiet and not _confirm_time(shutdown_t): return 1 config.AT_RUNNING_MINUTES = d_min try: vpn = vpntypes.VPN(args.vpn_type, cf, config.AT_RUNNING_MINUTES) except ValueError: log.error("this vpn type is broken") return 1 if vpn.needs_post_configure() and not args.identity_file: log.error("vpn type {} requires sshing, so an identity file (private key) is required".format(args.vpn_type)) parser.print_usage() return 1 host = launch(vpn, args.region) if host is not None: configure(host, vpn) log.info("\nephemeral vpn summoned\n") info = vpn.human_readable_data() info['Hostname'] = host longest = max(map(len, info.keys())) fmt = "{0:>{width}}: {1}" for k,v in info.iteritems(): log.info(fmt.format(k, v, width=longest)) # a new line for aesthetics log.info('\n') return 0
def create_archive(self, archive_name, dest=None): if not len(self.nsds): self.log.error("Did not find any NSDs to export") return if dest is None: dest = tempfile.mkdtemp() prevdir = os.getcwd() if not os.path.exists(dest): os.makedirs(dest) try: # Convert each NSD to a TOSCA template for nsd_id in self.nsds: # Not passing the dest dir to prevent clash in case # multiple export of the same desc happens self.create_csar(nsd_id) except Exception as e: msg = "Exception converting NSD {}: {}".format(nsd_id, e) self.log.exception(e) raise YangTranslateNsdError(msg) os.chdir(dest) try: if archive_name.endswith(".zip"): archive_name = archive_name[:-4] archive_path = os.path.join(dest, archive_name) # Construct a zip of the csar archives zip_name = '{}.zip'.format(archive_path) if len(self.csars) == 1: # Only 1 TOSCA template, just rename csar if required if self.csars[0] != zip_name: mv_cmd = "mv {} {}".format(self.csars[0], zip_name) subprocess.check_call(mv_cmd, shell=True, stdout=subprocess.DEVNULL) # Remove the temporary directory created shutil.rmtree(os.path.dirname(self.csars[0])) else: with zipfile.ZipFile('{}.partial'.format(zip_name), 'w') as zf: for csar in self.csars: # Move file to the current dest dir if os.path.dirname(csar) != dest: file_mv = "mv {} {}".format(csar, dest) subprocess.check_call(file_mv, shell=True, stdout=subprocess.DEVNULL) # Remove the temporary directory created shutil.rmtree(os.path.dirname(csar)) csar_f = os.basename(csar) # Now add to the archive zf.write(csar_f) # Remove the csar file os.remove(csar_f) # Rename archive to final name mv_cmd = "mv {0}.partial {0}".format(zip_name) subprocess.check_call(mv_cmd, shell=True, stdout=subprocess.DEVNULL) return zip_name except Exception as e: msg = "Creating CSAR archive failed: {0}".format(e) self.log.exception(e) raise YangTranslateError(msg) finally: os.chdir(prevdir)