def findsyspy(): """ :return: system python executable """ if not in_venv(): return sys.executable python = basename(realpath(sys.executable)) prefix = None if HAS_ORIG_PREFIX_TXT: with open(ORIG_PREFIX_TXT) as op: prefix = op.read() elif HAS_PY_VENV_CFG: prefix = getattr(sys, "_home") if not prefix: return None for folder in os.environ['PATH'].split(os.pathsep): if folder and \ normpath(normcase(folder)).startswith(normcase(normpath(prefix))) and \ isfile(join(folder, python)): return join(folder, python) # OSX: Homebrew doesn't leave python in the PATH if isfile(join(prefix, "bin", python)): return join(prefix, "bin", python)
def stdlib_functions(): A = UnexpectedClass() acos(A) # Noncompliant B = datetime.tzinfo() B.tzname( 42 ) # Noncompliant {{Change this argument; Function "tzname" expects a different type}} # ^^ select.select([], [], [], 0) time.sleep(1) # OK x = time.gmtime(int(time.time())) x = time.gmtime(secs=int(time.time())) time.sleep(True) # OK, converted to 1 time.sleep(1j) # FN, considered duck type compatible genericpath.isfile("some/path") genericpath.isfile(42) # Noncompliant my_list = [1, 2, 3] _heapq.heapify( 42 ) # Noncompliant {{Change this argument; Function "heapify" expects a different type}} # ^^ _heapq.heapify(my_list) imap4 = imaplib.IMAP4() imap4.setannotation(42) # FN, we do not handle variadic parameters imap4.setannotation("some string") # OK str_tuple = "string", "another string" imap4.setannotation(str_tuple) # OK
def print_page(): if not _printer: return "there is no printer", 400 if request.method == 'POST': if request.files and 'uploadfile' in request.files: file = request.files['uploadfile'] if file and beaglebone_helpers.allowed_file(file.filename): filename = secure_filename(file.filename) upload_path = os.path.join(app.config['UPLOAD_FOLDER'], filename) try: _logger.info("Saving file %s to %s", filename, upload_path) file.save(upload_path) except: _logger.warn("unable to save file %s to %s", filename, upload_path) elif 'printfile' in request.form: filename = request.form['printfile'] file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename) if isfile(file_path) and beaglebone_helpers.allowed_file(filename): _printer.prepared_file = file_path _logger.info("Printing %s", _printer.prepared_file) global _print_thread _print_thread = GCodePrintThread(_printer.prepared_file, _printer, None) _print_thread.start() template_dictionary = templating_defaults() files = [f for f in listdir(app.config['UPLOAD_FOLDER']) if isfile(app.config['UPLOAD_FOLDER'] + "/" + f) and fnmatch.fnmatch(f, '*.gcode')] #todo would like to http://stackoverflow.com/questions/6591931/getting-file-size-in-python #http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size #http://stackoverflow.com/questions/237079/how-to-get-file-creation-modification-date-times-in-python template_dictionary['files'] = files if _printer.prepared_file: template_dictionary['print_file'] = _printer.prepared_file.rsplit('/', 1)[1] return render_template("print.html", **template_dictionary)
def _read_pickled(self, etag, mod_stamp): if not isfile(self.cache_path_pickle) or not isfile(self.cache_path_json): # Don't trust pickled data if there is no accompanying json data return None try: if isfile(self.cache_path_pickle): log.debug("found pickle file %s", self.cache_path_pickle) with open(self.cache_path_pickle, 'rb') as fh: _pickled_state = pickle.load(fh) except Exception as e: log.debug("Failed to load pickled repodata.", exc_info=True) rm_rf(self.cache_path_pickle) return None def _check_pickled_valid(): yield _pickled_state.get('_url') == self.url_w_credentials yield _pickled_state.get('_schannel') == self.channel.canonical_name yield _pickled_state.get('_add_pip') == context.add_pip_as_python_dependency yield _pickled_state.get('_mod') == mod_stamp yield _pickled_state.get('_etag') == etag yield _pickled_state.get('_pickle_version') == REPODATA_PICKLE_VERSION if not all(_check_pickled_valid()): log.debug("Pickle load validation failed for %s at %s.", self.url_w_subdir, self.cache_path_json) return None return _pickled_state
def _read_pickled(self, etag, mod_stamp): if not isfile(self.cache_path_pickle) or not isfile( self.cache_path_json): # Don't trust pickled data if there is no accompanying json data return None try: if isfile(self.cache_path_pickle): log.debug("found pickle file %s", self.cache_path_pickle) with open(self.cache_path_pickle, 'rb') as fh: _pickled_state = pickle.load(fh) except Exception as e: log.debug("Failed to load pickled repodata.", exc_info=True) rm_rf(self.cache_path_pickle) return None def _check_pickled_valid(): yield _pickled_state.get('_url') == self.url_w_credentials yield _pickled_state.get( '_schannel') == self.channel.canonical_name yield _pickled_state.get( '_add_pip') == context.add_pip_as_python_dependency yield _pickled_state.get('_mod') == mod_stamp yield _pickled_state.get('_etag') == etag yield _pickled_state.get( '_pickle_version') == REPODATA_PICKLE_VERSION if not all(_check_pickled_valid()): log.debug("Pickle load validation failed for %s at %s.", self.url_w_subdir, self.cache_path_json) return None return _pickled_state
def read_pickled_repodata(cache_path, channel_url, schannel, priority, etag, mod_stamp): pickle_path = get_pickle_path(cache_path) # Don't trust pickled data if there is no accompanying json data if not isfile(pickle_path) or not isfile(cache_path): return None try: if isfile(pickle_path): log.debug("found pickle file %s", pickle_path) with open(pickle_path, 'rb') as f: repodata = pickle.load(f) except Exception as e: import traceback log.debug("Failed to load pickled repodata.\n%s", traceback.format_exc()) rm_rf(pickle_path) return None def _check_pickled_valid(): yield repodata.get('_url') == channel_url yield repodata.get('_schannel') == schannel yield repodata.get('_add_pip') == context.add_pip_as_python_dependency yield repodata.get('_mod') == mod_stamp yield repodata.get('_etag') == etag yield repodata.get('_pickle_version') == REPODATA_PICKLE_VERSION if not all(_check_pickled_valid()): return None if int(repodata['_priority']) != priority: log.debug("setting priority for %s to '%d'", repodata.get('_url'), priority) repodata['_priority']._priority = priority return repodata
def _build(self, locale, messages, path, pathGlobal=None): ''' Builds a catalog based on the provided locale paths, the path is used as the main source any messages that are not found in path locale but are part of messages will attempt to be extracted from the global path locale. @param locale: Locale The locale. @param messages: Iterable(Message) The messages to build the PO file on. @param path: string The path of the targeted PO file from the locale repository. @param pathGlobal: string|None The path of the global PO file from the locale repository. @return: file like object File like object that contains the PO file content ''' assert isinstance(locale, Locale), 'Invalid locale %s' % locale assert isinstance(messages, Iterable), 'Invalid messages %s' % messages assert isinstance(path, str), 'Invalid path %s' % path assert pathGlobal is None or isinstance(pathGlobal, str), 'Invalid global path %s' % pathGlobal if isfile(path): with open(path) as fObj: catalog = read_po(fObj, locale) else: catalog = Catalog(locale, creation_date=datetime.now(), **self.catalog_config) if pathGlobal and isfile(pathGlobal): with open(pathGlobal) as fObj: catalogGlobal = read_po(fObj, locale) else: catalogGlobal = None self._processCatalog(catalog, messages, fallBack=catalogGlobal) catalog.revision_date = datetime.now() return catalog
def test_isfile(self): self.assertIs(genericpath.isfile(support.TESTFN), False) f = open(support.TESTFN, "wb") try: f.write(b"foo") f.close() self.assertIs(genericpath.isfile(support.TESTFN), True) os.remove(support.TESTFN) os.mkdir(support.TESTFN) self.assertIs(genericpath.isfile(support.TESTFN), False) os.rmdir(support.TESTFN) finally: if not f.close(): f.close() try: os.remove(support.TESTFN) except os.error: pass try: os.rmdir(support.TESTFN) except os.error: pass self.assertRaises(TypeError, genericpath.isdir) def test_samefile(self): f = open(support.TESTFN + "1", "wb") try: f.write(b"foo") f.close() self.assertIs( genericpath.samefile(support.TESTFN + "1", support.TESTFN + "1"), True) # If we don't have links, assume that os.stat doesn't return resonable # inode information and thus, that samefile() doesn't work if hasattr(os, "symlink"): os.symlink(support.TESTFN + "1", support.TESTFN + "2") self.assertIs( genericpath.samefile(support.TESTFN + "1", support.TESTFN + "2"), True) os.remove(support.TESTFN + "2") f = open(support.TESTFN + "2", "wb") f.write(b"bar") f.close() self.assertIs( genericpath.samefile(support.TESTFN + "1", support.TESTFN + "2"), False) finally: if not f.close(): f.close() try: os.remove(support.TESTFN + "1") except os.error: pass try: os.remove(support.TESTFN + "2") except os.error: pass self.assertRaises(TypeError, genericpath.samefile)
def run_annotators(cnf, vcf_fpath): work_dir = cnf['work_dir'] annotated = False if 'gatk' in cnf: annotated = True vcf_fpath = _gatk(cnf, vcf_fpath, cnf.get('bam'), work_dir) if 'dbsnp' in cnf: annotated = True vcf_fpath = _snpsift_annotate(cnf, cnf['dbsnp'], 'dbsnp', vcf_fpath, work_dir) if 'cosmic' in cnf: annotated = True vcf_fpath = _snpsift_annotate(cnf, cnf['cosmic'], 'cosmic', vcf_fpath, work_dir) if 'custom_vcfs' in cnf: for dbname, custom_conf in cnf['custom_vcfs'].items(): annotated = True vcf_fpath = _snpsift_annotate( cnf, custom_conf, dbname, vcf_fpath, work_dir) if 'dbnsfp' in cnf: annotated = True vcf_fpath = _snpsift_db_nsfp(cnf, vcf_fpath, work_dir) if 'snpeff' in cnf: annotated = True _remove_annotation(cnf, 'EFF', vcf_fpath, work_dir) vcf_fpath, summary_fpath, genes_fpath = _snpeff(cnf, vcf_fpath, work_dir) if isfile(join(cnf['output_dir'], summary_fpath)): os.remove(join(cnf['output_dir'], summary_fpath)) if isfile(join(cnf['output_dir'], genes_fpath)): os.remove(join(cnf['output_dir'], genes_fpath)) if file_exists(summary_fpath): shutil.move(summary_fpath, cnf['output_dir']) if file_exists(genes_fpath): shutil.move(genes_fpath, cnf['output_dir']) if cnf.get('tracks'): for track in cnf['tracks']: annotated = True vcf_fpath = _tracks(cnf, track, vcf_fpath, work_dir) if annotated: vcf_fpath = _filter_fields(cnf, vcf_fpath, work_dir) # Copying final VCF final_vcf_fname = add_suffix(basename(cnf['vcf']), 'anno') final_vcf_fpath = join(cnf['output_dir'], final_vcf_fname) if isfile(final_vcf_fpath): os.remove(final_vcf_fpath) shutil.copyfile(vcf_fpath, final_vcf_fpath) return final_vcf_fpath else: info('No annotations were run on ' + vcf_fpath + '. Please, specify some in run_info.') return None
def ida_extract(path, args, procedure_name): """ returns work_path where the results were stored """ # Run IDA headless and extract the procedures extract_script_path = path_join(dirname(realpath(__file__)), "py2", 'bin_extract.py') extract_command = 'cd {};TVHEADLESS=1 {} -B -S"{}{}" {} > {}'. \ format(dirname(path), args['idal64_path'], extract_script_path, "" if procedure_name is None else " " + procedure_name, basename(path), FNULL.name) r, output = getstatusoutput(extract_command) if r != 0: logging.warning( "IDA command {} had errors. Giving it another (last) chance.". format(extract_command)) r, output = getstatusoutput(extract_command) if r != 0: raise Exception( "IDA command {} returned error code {}, even after retry!". format(extract_command, r)) work_path = dirname(path) # float ida critical errors ida_log_file_path = path_join(work_path, log_file_name) if isfile(ida_log_file_path): with open(ida_log_file_path) as ida_log_file: for line in ida_log_file.readlines(): if line.startswith("CRITICAL"): logging.critical( "file={}, IDA Critical Message - {}".format( path, line)) raise RuntimeError( "Error in IDA run for - {}".format(path)) else: logging.warning("IDA log file not found! ({})".format(path)) if not args['keep_temps']: for temporary in [ path_join(dirname(path), f) for f in listdir(dirname(path)) ]: if temporary.endswith( IndexedExeFile.get_filename()) or temporary.endswith( IndexedProcedure.get_filename_suffix()): continue if temporary.endswith("i64") or temporary == path: continue if temporary.endswith(extern_dump_file_name): continue # ida will only make tmp files no dirs, so we can do this.. if isfile(temporary): remove(temporary) return work_path
def bgzip_and_tabix(cnf, vcf_fpath, tabix_parameters='', **kwargs): gzipped_fpath = join(vcf_fpath + '.gz') tbi_fpath = gzipped_fpath + '.tbi' if cnf.reuse_intermediate and \ file_exists(gzipped_fpath) and \ file_exists(tbi_fpath) and getctime(tbi_fpath) >= getctime(gzipped_fpath): info('Actual compressed VCF and index exist, reusing') return gzipped_fpath info('Compressing and tabixing VCF file, writing ' + gzipped_fpath + '(.tbi)') bgzip = get_system_path(cnf, 'bgzip') tabix = get_system_path(cnf, 'tabix') if not bgzip: err('Cannot index VCF because bgzip is not found in PATH or ' + cnf.sys_cnf) if not tabix: err('Cannot index VCF because tabix is not found in PATH or ' + cnf.sys_cnf) if not bgzip and not tabix: return vcf_fpath retrying = False while True: if isfile(tbi_fpath): os.remove(tbi_fpath) if isfile(vcf_fpath): if isfile(gzipped_fpath): os.remove(gzipped_fpath) info('BGzipping VCF') cmdline = '{bgzip} {vcf_fpath}'.format(**locals()) call(cnf, cmdline, None, **kwargs) else: if not verify_file(gzipped_fpath): err('Neither uncompressed ' + vcf_fpath + ' nor ' + gzipped_fpath + ' exist') return None info('Tabixing VCF') cmdline = '{tabix} {tabix_parameters} {gzipped_fpath}'.format(**locals()) exit_on_error = False if retrying: exit_on_error = True kwargs['exit_on_error'] = exit_on_error call(cnf, cmdline, **kwargs) if isfile(gzipped_fpath + '.tbi'): break if retrying: critical('Cannot tabix ' + vcf_fpath) if not isfile(vcf_fpath): call(cnf, 'gunzip ' + gzipped_fpath, None) retrying = True return gzipped_fpath
def in_venv(): """ :return: True if in running from a virtualenv Has to detect the case where the python binary is run directly, so VIRTUAL_ENV may not be set """ global _in_venv logger.debug("in_venv enter with interpreter %s", sys.executable) if _in_venv is not None: logger.debug("in_venv return cached value %s", _in_venv) return _in_venv if not (os.path.isfile(ORIG_PREFIX_TXT) or os.path.isfile(PY_VENV_CFG)): logger.debug("in_venv no orig_prefix_txt [%s]", ORIG_PREFIX_TXT) logger.debug("in_venv no py_venv_cfg [%s]", PY_VENV_CFG) # TODO - check this is actually valid ! _in_venv = False return _in_venv if 'VIRTUAL_ENV' in os.environ: logger.debug("in_venv VIRTUAL_ENV set.") _in_venv = True else: # Find first python in path ... if its not this one, # ...we are in a different python environment python = basename(sys.executable) for p in os.environ['PATH'].split(os.pathsep): py_path = join(p, python) if isfile(py_path): _in_venv = sys.executable != py_path logger.debug("in_venv: [%s], sys.executable is not first python in path [%s]", _in_venv, py_path) break return _in_venv
def select_game(): GAMES_DIR = 'games' DEFAULT_CONF_FILE = 'conf_file' games = [x for x in os.listdir(GAMES_DIR) if isdir(GAMES_DIR + "/" + x) and isfile(GAMES_DIR + "/" + x + "/" + DEFAULT_CONF_FILE)] if len(games) < 1: print("No games are present in the default directory (Do you have put them in the folder \'games\'?)") return False while True: print("Available games:") for i, game in enumerate(games): print("\t" + str(i) + ") " + str(game)) try: choosed = int(input('Select the puzzle that you want to solve: ')) print("You have chosen: " + str(games[choosed])) print() print("--------------------------------") try: return GAMES_DIR + os.sep + games[choosed] + os.sep, DEFAULT_CONF_FILE except Exception as ex: print('An error occurs :(') print("--------------------------------") print() except (ValueError, TypeError): return None, None
def adjust_proteomes(prot_fpaths, proteomes_dir, prot_id_field): if not isdir(proteomes_dir): mkdir(proteomes_dir) for proteome_fpath in prot_fpaths: prot_ids = set() records = [] taxon_id, ext = splitext(basename(proteome_fpath)) ext = '.fasta' for seq in SeqIO.parse(proteome_fpath, 'fasta'): fields = seq.id.replace('|', ' ').split() if len(fields) > prot_id_field: prot_id = fields[prot_id_field] elif len(fields) > 0: prot_id = fields[-1] else: log.error('Incorrect fasta id: ' + str(seq.id)) return 1 if len(prot_id) > 30: prot_id = prot_id[:17] + '...' + prot_id[-10:] if prot_id in prot_ids: log.error('Fasta {proteome_fpath} contains duplicate id: ' '{prot_id}'.format(**locals())) return 1 prot_ids.add(prot_id) seq.id = taxon_id + '|' + prot_id records.append(seq) out_fpath = join(proteomes_dir, taxon_id + ext) if isfile(out_fpath): remove(out_fpath) SeqIO.write(records, out_fpath, 'fasta') return 0
def resample_allfiles( results_dir: Optional[str] = None, resampled_dir: Optional[str] = None ) -> int: """Resample all files inside results_dir folder. Parameters ---------- results_dir: str Directory containing results to resample. resampled_dir: str Directory to save resampled results. """ if results_dir is None: results_dir = eniric.paths["results"] # type: ignore if resampled_dir is None: resampled_dir = eniric.paths["resampled"] # type: ignore # Getting a list of all the files onlyfiles = [f for f in os.listdir(results_dir) if isfile(join(results_dir, f))] for spectrum_file in onlyfiles: if spectrum_file.endswith(".dat"): resampler( spectrum_file, results_dir=results_dir, resampled_dir=resampled_dir ) return 0
def run_manifests(stack_dir): images = read_var('IMAGES_BUILT') image_deps = json2series(read_dict('image-dependency.json'), 'dep', 'image') store_series(image_deps, 'image-dependency') # Write image dependency table to wiki dep_table_fp = 'wiki/Image Dependency.md' if isfile(dep_table_fp): old_csv = strip_csv_from_md(dep_table_fp) csv_concat(old_csv, 'artifacts/image-dependency.csv', 'artifacts/image-dependency-updated.csv') csv_embed_markdown('artifacts/image-dependency-updated.csv', dep_table_fp, 'Image Dependency') else: csv_embed_markdown('artifacts/image-dependency.csv', dep_table_fp, 'Image Dependency') specs = get_specs(path.join(stack_dir, 'spec.yml')) for image in images: keys = list(filter(lambda x: x in image, specs['images'])) assert len(keys) == 1 image_key = keys[0] print('Running image manifest for', image) run_report(specs, image_key, image=image) insert_history('wiki/Home.md')
def _show_dir(path): html = '' for item in os.listdir(settings.TINYMCE_IMAGES_STORAGE_ROOT + path): if isfile(settings.TINYMCE_IMAGES_STORAGE_ROOT + path + item): file = ImageFile(settings.TINYMCE_IMAGES_STORAGE_ROOT, path + item) html += render_to_string('tinymce_images_filelist.html', {'image' : file}) return html
def read_data_from_csv(csv_path, sort=True): """ Read data from CSV into DataFrame. :param csv_path: absolute path to CSV file :param sort: whether to sort the files by file size :return: df: pd.DataFrame containing the CSV data duration: integer representing the total length of the audio by summing over the 'wav_length' column from the CSV. If such a column is not present, the value returned is math.inf """ if not isfile(csv_path): print(f'ERROR: CSV file {csv_path} does not exist!', file=sys.stderr) exit(0) print(f'Reading samples from {csv_path}...', end='') df = pd.read_csv(csv_path, sep=',', encoding='utf-8') print(f'done! ({len(df.index)} samples', end='') duration = math.inf if 'wav_length' in df: duration = df['wav_length'].sum() avg_duration = df['wav_length'].mean() print( f', {timedelta(seconds=duration)}, Ø audio length: {avg_duration:.4f} seconds)' ) else: print(')') if sort: df = df.sort_values(by='wav_filesize', ascending=True) avg_trans_length = np.mean([len(trans) for trans in df['transcript']]) print(f'average transcript length: {avg_trans_length}') return df.reset_index(drop=True), duration
def in_venv(): """ :return: True if in running from a virtualenv Has to detect the case where the python binary is run directly, so VIRTUAL_ENV may not be set """ global _in_venv if _in_venv is not None: return _in_venv if not os.path.isfile(orig_prefix): logger.debug("in_venv no orig_prefix [%s]", orig_prefix) # TODO - check this is actually valid ! _in_venv = False return _in_venv if 'VIRTUAL_ENV' in os.environ: logger.debug("in_venv VIRTUAL_ENV set.") _in_venv = True else: # Find first python in path ... if its not this one, # ...we are in a different python python = basename(sys.executable) for p in os.environ['PATH'].split(os.pathsep): py_path = join(p, python) if isfile(py_path): logger.debug("in_venv py_at [%s] return: %s", (py_path, sys.executable != py_path)) _in_venv = sys.executable != py_path break return _in_venv
def read(self): for node in os.listdir(join(self.root, self.path)): full_path = join(self.path, node) if isdir(join(self.root, full_path)): self.folders.append(Folder(self.root, full_path)) if isfile(join(self.root, full_path)): self.files.append(File(self.root, full_path))
def match_or_trust(self, host, der_encoded_certificate): base64_encoded_certificate = b64encode(der_encoded_certificate) if isfile(self.path): with open(self.path) as f_in: for line in f_in: known_host, _, known_cert = line.strip().partition(":") known_cert = known_cert.encode("utf-8") if host == known_host: return base64_encoded_certificate == known_cert # First use (no hosts match) try: makedirs(dirname(self.path)) except OSError: pass f_out = os_open(self.path, O_CREAT | O_APPEND | O_WRONLY, 0o600) # TODO: Windows if isinstance(host, bytes): os_write(f_out, host) else: os_write(f_out, host.encode("utf-8")) os_write(f_out, b":") os_write(f_out, base64_encoded_certificate) os_write(f_out, b"\n") os_close(f_out) return True
def tenant_bills_to_docx(self): """ Compose a single bill for a tenant_id, save it to disk. returns a string containing the path to the saved file. """ for t in self.rtp.tenantBillList: if not self.rtp.bill_tenant_0: if t.tenant_id == 0: continue t_bill = MailMerge(self.rtp.pdf_docx_template) t_bill.merge( name=format_values(t.tenant_name), month_year=format_values("{} {}".format( self.rtp.month_str, self.rtp.year)), total=format_values(t.charge_total), date=format_values( datetime.datetime.now().strftime('%Y %B %d')), room_rate=format_values(t.charge_room), gas=format_values(t.charge_gas), internet=format_values(t.charge_internet), electricity=format_values(t.charge_electricity), recurring=format_values(t.charge_recurring), other=format_values(t.charge_other), memo_gas=format_values(t.memo_gas), memo_internet=format_values(t.memo_internet), memo_electricity=format_values(t.memo_electricity), memo_recurring=format_values(t.memo_other), memo_other=format_values(t.memo_other), ) docx = self.rtp.google_path + "bills\\" + self.__format_file_name( t, "docx") if isfile(docx): remove(docx) t_bill.write(docx) t.docx_long_name = docx self.logger.debug("Created docx_long_name file: {}".format(docx))
def addpackage(sitedir, pthfile, known_dirs=None): """ Wrapper for site.addpackage Try and work out which directories are added by the .pth and add them to the known_dirs set """ known_dirs = set(known_dirs or []) with open(join(sitedir, pthfile)) as f: for n, line in enumerate(f): if line.startswith("#"): continue line = line.rstrip() if line: if line.startswith(("import ", "import\t")): exec (line, globals(), locals()) continue else: p_rel = join(sitedir, line) p_abs = abspath(line) if isdir(p_rel): os.environ['PATH'] += env_t(os.pathsep + p_rel) sys.path.append(p_rel) added_dirs.add(p_rel) elif isdir(p_abs): os.environ['PATH'] += env_t(os.pathsep + p_abs) sys.path.append(p_abs) added_dirs.add(p_abs) if isfile(pthfile): site.addpackage(sitedir, pthfile, known_dirs) else: logging.debug("pth file '%s' not found")
def _lastModified(self, locale, component=None, plugin=None): ''' Provides the last modification time stamp for the provided locale. You can specify the component id in order to get the last modification for the component domain, or plugin or either to get the global domain modification. @param locale: Locale The locale to get the last modification for. @param component: string|None The component id to get the last modification for. @param plugin: string|None The plugin id to get the last modification for. @return: datetime|None The last modification time stamp, None if there is no such time stamp available. ''' assert isinstance(locale, Locale), 'Invalid locale %s' % locale assert not(component and plugin), 'Cannot process a component id %s and a plugin id %s' % (component, plugin) q = QSource() q.lastModified.orderDesc() if component: q.component = component elif plugin: q.plugin = plugin sources = self.sourceService.getAll(0, 1, q=q) try: lastModified = next(iter(sources)).LastModified except StopIteration: lastModified = None path = self._filePath(locale, component, plugin) if isfile(path): lastModified = max(lastModified, datetime.fromtimestamp(os.stat(path).st_mtime)) return lastModified
def add_env_path(path): # Add package folder to searching path # Search the dir that contains file '__dbreport__.py' for dirname in work_all_path(path): if isfile(dirname + os.sep + '__dbreport__.py') and dirname not in sys.path: sys.path.append(dirname)
def _lastModified(self, locale, component=None, plugin=None): ''' Provides the last modification time stamp for the provided locale. You can specify the component id in order to get the last modification for the component domain, or plugin or either to get the global domain modification. @param locale: Locale The locale to get the last modification for. @param component: string|None The component id to get the last modification for. @param plugin: string|None The plugin id to get the last modification for. @return: datetime|None The last modification time stamp, None if there is no such time stamp available. ''' assert isinstance(locale, Locale), 'Invalid locale %s' % locale assert not ( component and plugin ), 'Cannot process a component id %s and a plugin id %s' % (component, plugin) q = QSource() q.lastModified.orderDesc() if component: q.component = component elif plugin: q.plugin = plugin sources = self.sourceService.getAll(0, 1, q=q) try: lastModified = next(iter(sources)).LastModified except StopIteration: lastModified = None path = self._filePath(locale, component, plugin) if isfile(path): lastModified = max(lastModified, datetime.fromtimestamp(os.stat(path).st_mtime)) return lastModified
def main(argv1, argv2): # load necessary configuration # argv1 -- output file path # argv2 -- corpus path p_model = load_model( "/opt/gsda/EUPEG/Geoparsers/camCoder/data/weights" ) # weights to be downloaded from Cambridge Uni repo, see GitHub. p_word_to_index = cPickle.load( open(u"/opt/gsda/EUPEG/Geoparsers/camCoder/codes/data/words2index.pkl") ) # This is the vocabulary file # Example usage of the geoparse function below reading from a directory and parsing all files. directory = unicode(str(argv2), 'utf-8') files = [f for f in listdir(directory) if isfile(directory + f)] # output_file = open(argv1, 'w+') with open(argv1, "w") as wf: for i in range(0, len(files)): # print i full_line = "" for line in codecs.open(directory + str(i), encoding="utf-8", errors='ignore'): full_line = full_line + line article_parsing = geoparse(full_line, "", p_model, p_word_to_index) # write result into the output wf.write(article_parsing) wf.write('\n')
def __init__(self): self.cache = "parscit_cache" if "PARSCIT" not in environ: raise ValueError("Enviroment variable PARSCIT must point to PARSCIT source") self.script = join(environ["PARSCIT"], "bin", "citeExtract.pl") if not isfile(self.script): raise ValueError()
def migrate_copy2_remote(): result_file = open("/home/work/data/collie/local_modules/results.log", "wb+") webhdfs = WebHdfs() for version_id in range(0, 2000): try: base_path = "/home/work/data/collie/local_modules/" + str( version_id) + "/" files = [ f for f in listdir(base_path) if isfile(join(base_path, f)) ] if len(files) <= 0: continue file_name = files[0] version_file = open(join(base_path, file_name), 'rb') checksum = checksum_file(version_file) file_name_new = file_name + "_" + checksum os.rename(join(base_path, file_name), join(base_path, file_name_new)) remote_path = "/user/h_sns/collie/modules/" webhdfs.copyFromLocal(join(base_path, file_name_new), remote_path) result_file.write( str(version_id) + "\t" + remote_path + file_name_new + "\n") print "%d is saving to remote %s" % (version_id, remote_path + file_name_new) except IOError: continue except OSError: continue
def __init__(self, logFile='log.txt'): self.logFile = logFile self.sightingFile = None self.ariesFile = None self.starFile = None self.sightingError = 0 self.sighting_tuples = [] if not isinstance(logFile, str): raise ValueError('Fix.__init__: "logFile" is a string') if (len(logFile) < 1): raise ValueError( 'Fix.__init__: "logFile" should have a length .GE. 1') if not isfile(logFile): try: log = open(logFile, 'w') except: raise ValueError('Fix.__init__: "logFile" can not be created') else: try: log = open(logFile, 'a') except: raise ValueError( 'Fix.__init__: "logFile" can not be opened for appending') logEntry = self.message("Log file:\t" + os.path.abspath(self.logFile)) try: log.write(logEntry) except: raise ValueError('Fix.__init__: "logFile" can not be appended') log.close()
def catch_all(path): current_app.logger.debug("Working in: %s", os.environ.get("SERVER_STATIC_FOLDER", "")) current_app.logger.debug("FALLBACK for path: %s", path) # check compression compress = os.environ["COMPRESS"].lower() == "true" if compress and (path.endswith(".js") or path.endswith(".css")): current_app.logger.debug("Use compression") path += ".gz" else: compress = False # detect language from url path (en|hu) languages = os.environ["LANGUAGES"].split(" ") result = re.search("(" + "|".join(languages) + ")", path) language = result.group(0) if result else "" current_app.logger.debug("Language: %s from %s", language or "No language in URL", languages) if language == "en": path = path.replace("en/", "") current_app.logger.debug("FALLBACK for path processed: %s", path) # return with file if exists current_app.logger.debug("Checking for %s", path) if isfile(join(current_app.config["WEBAPP_SOURCE"], path)): current_app.logger.debug("Path exists without language: %s", path) response = send_from_directory(current_app.config["WEBAPP_SOURCE"], path) if compress: response.headers["Content-Encoding"] = "gzip" return response elif language and isfile( join(current_app.config["WEBAPP_SOURCE"], language, "index.html")): current_app.logger.debug("Path exists with language: %s", join(language, "index.html")) return send_from_directory( join(current_app.config["WEBAPP_SOURCE"], language), "index.html") # or return with the index file current_app.logger.debug("INDEX without language: %s", join(language, "index.html")) return send_from_directory(current_app.config["WEBAPP_SOURCE"], "index.html")
def initialize(): if isfile(config.case_db_path): if config.reinitialize_case_db_on_startup: case_db.tearDown() remove(config.case_db_path) case_db.create() else: case_db.create()
def __init__(self): self.cache = "parscit_cache" if "PARSCIT" not in environ: raise ValueError( "Enviroment variable PARSCIT must point to PARSCIT source") self.script = join(environ["PARSCIT"], "bin", "citeExtract.pl") if not isfile(self.script): raise ValueError()
def getfiles(self, src_dir): files = [] for f in os.listdir(src_dir): if isfile(os.path.join(src_dir, f)): if not f.startswith('.'): files.append(os.path.join(src_dir, f)) return files
def get(): if exists(VersionManager.__location) and isfile(VersionManager.__location): try: with open(VersionManager.__location) as f: return json.load(f) except: LOG.error("Failed to load version from '%s'" % VersionManager.__location) return {"coreVersion": None, "enclosureVersion": None}
def __load(config, location): if exists(location) and isfile(location): try: config.update(load_commented_json(location)) LOG.debug("Configuration '%s' loaded" % location) except Exception, e: LOG.error("Error loading configuration '%s'" % location) LOG.error(repr(e))
def __load(config, location): if exists(location) and isfile(location): try: with open(location) as f: config.update(json.load(f)) LOG.debug("Configuration '%s' loaded" % location) except Exception, e: LOG.error("Error loading configuration '%s'" % location) LOG.error(repr(e))
def index_image_folder(folder_path): for f in listdir(folder_path): path = join(folder_path, f) if isfile(path): is_image = any(path.endswith("." + ext) for ext in config.IMAGE_EXTENSIONS) if is_image: # check extension, only index images index_image(folder_path=folder_path, file_name=f) elif isdir(path): index_image_folder(path)
def __load(config, location): if exists(location) and isfile(location): try: cobj = ConfigObj(location) config = ConfigurationLoader.__merge(config, cobj) logger.debug("Configuration '%s' loaded" % location) except Exception, e: logger.error("Error loading configuration '%s'" % location) logger.error(repr(e))
def process_item(self, item, spider): if 'prices' in spider.name: self._register(item) if isfile(self._csv_file): self._write() else: self._initialize() self.process_item(item, spider) return item
def __init__(self): my_cache_file_path = LibExpRepo.my_cache if not isfile(my_cache_file_path) and isfile(my_cache_file_path + ".zip"): import zipfile logging.warning("Extracting LIBEXP zip file") with zipfile.ZipFile(my_cache_file_path + ".zip", 'r') as zip_ref: zip_ref.extractall(".") if isfile(my_cache_file_path): with open(my_cache_file_path, "rb") as cache_file_in: self._repo = pickle_load(cache_file_in) logging.warning( "Loaded LIBEXP from cache, (has ={} records)".format( len(list(self._repo.keys())))) return
def check_hdf5(self, reload=False): if reload: Hdf5Creator().create_hdf5() elif isfile(self.hdf5_path): if not PytablesHdf5Manager( self.hdf5_path).check_exist_dataframes(DS_ARRAY): Hdf5Creator().create_hdf5() else: Hdf5Creator().create_hdf5()
def find_project_data(p, levels_to_project_traits): p = split(abspath(p)) if not p[1]: return '', levels_to_project_traits if isfile(join(p[0], 'data', database)): return p[0], levels_to_project_traits else: return find_project_data(p[0], levels_to_project_traits+1)
def test_settings_nonworld(): """ Check that settings_local.py exists, is somewhat hidden and has a secret key and database settings. """ pth = join(settings.BASE_DIR, 'source', 'local.py') assert isfile(pth), 'you should have a local settings file with secret settings, located at "{0:s}"'.format(pth) mode = stat(pth).st_mode assert not mode & S_IROTH, 'local settings should not be world-readable (file "{0:s}")'.format(pth) assert not mode & S_IWOTH, 'local settings should not be world-writable (file "{0:s}")'.format(pth) assert not mode & S_IXOTH, 'local settings should not be world-executable (file "{0:s}")'.format(pth)
def get(): if (exists(VersionManager.__location) and isfile(VersionManager.__location)): try: with open(VersionManager.__location) as f: return json.load(f) except: LOG.error("Failed to load version from '%s'" % VersionManager.__location) return {"coreVersion": None, "enclosureVersion": None}
def get(): data_dir = expanduser(Configuration.get()['data_dir']) version_file = join(data_dir, 'version.json') if exists(version_file) and isfile(version_file): try: with open(version_file) as f: return json.load(f) except Exception: LOG.error("Failed to load version from '%s'" % version_file) return {"coreVersion": None, "enclosureVersion": None}
def check_common_args(params): if params.start_from == 'uselog': if not isfile(join(params.out_dir, config.log_fname)): arg_parse_error( 'No %s in %s. Either check your path, or ' 'change the --start-from option' % (config.log_fname, params.out_dir)) if params.start_from: params.overwrite = True
def getFilesFromFolder(infiles): """ :param infiles: :return: """ only_files = [f for f in listdir(infiles) if isfile(join(infiles, f))] if '.DS_Store' in only_files: only_files.__delitem__(only_files.index('.DS_Store')) return only_files
def classpath_string(self, scope): resolved_classpath_file = os.path.join(self.package_cache, "%s-%s-%s" % (self.group_id, self.artifact_id, self.version)) if not isfile(resolved_classpath_file): print "Resolving ivy dependency %s" % self subprocess.check_call(["java", "-jar", self.ivy_executable, "-dependency", self.group_id, self.artifact_id, self.version, "-cachepath", resolved_classpath_file]) with open(resolved_classpath_file) as f: return f.read().replace("\n", "")
def read(self): for entry in os.listdir(join(self.root, self.path)): full_path = join(self.path, entry) if isdir(join(self.root, full_path)): node = Folder(self.root, full_path) self.folders.append(node) if isfile(join(self.root, full_path)): node = File(self.root, full_path) self.files.append(node) self.nodes.append(node) self.nodes.sort(key=lambda node: node.key())
def __init__(self, dependent_id, dependency_id): super(IvyDependencyJob, self).__init__(dependent_id, {}) self.ivy_executable = os.path.expanduser("~/.bacon.d/ivy.jar") split_dep = dependency_id.split(":") self.group_id = split_dep[0] self.artifact_id = split_dep[1] self.version = split_dep[2] if not isfile(self.ivy_executable): print "Downloading ivy..." urllib.urlretrieve("http://central.maven.org/maven2/org/apache/ivy/ivy/2.3.0/ivy-2.3.0.jar", self.ivy_executable)
def get_spiders_files(spiders_directory=None): """ Return list of filename corresponding to JobSpider :param spiders_directory: Path where search JobSpiders :return: list of filename """ if spiders_directory is None: spiders_directory = dirname(__file__) + '/spiders/' return [file for file in glob.glob(spiders_directory + "/*.py") if isfile(file) and not file.endswith('__init__.py')]
def process_transcoding((flac_file, flac_root_path, mp3_target_path)): try: target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file) if not isdir(dirname(target_mp3_file)): try: os.makedirs(dirname(target_mp3_file)) except OSError: pass # other thread might have been faster if isfile(target_mp3_file) and tags_are_equals(flac_file, target_mp3_file): LOGGER.info('skipping %r as target mp3 file exists and seems to have the same tags', flac_file) else: transcode(flac_file, target_mp3_file) except Exception as e: LOGGER.exception('error during the transcoding of %r : %s' % (flac_file, e))
def compress(source_dir, dest_file): relroot = abspath(join(source_dir, "..")) with ZipFile(dest_file, 'w', compression=ZIP_DEFLATED) as zf: for root, dirs, files in walk(source_dir): zf.write(root, relpath(root, relroot)) for f in files: fname = join(root, f) if isfile(fname): arcname = join(relpath(root, relroot), f) #if '.cpan' not in fname and 'perl_modules' not in fname: print ' ' + arcname # join(relpath(root, directory), f) zf.write(fname, arcname)