def findsyspy():
"""
:return: system python executable
"""
if not in_venv():
return sys.executable
python = basename(realpath(sys.executable))
prefix = None
if HAS_ORIG_PREFIX_TXT:
with open(ORIG_PREFIX_TXT) as op:
prefix = op.read()
elif HAS_PY_VENV_CFG:
prefix = getattr(sys, "_home")
if not prefix:
return None
for folder in os.environ['PATH'].split(os.pathsep):
if folder and \
normpath(normcase(folder)).startswith(normcase(normpath(prefix))) and \
isfile(join(folder, python)):
return join(folder, python)
# OSX: Homebrew doesn't leave python in the PATH
if isfile(join(prefix, "bin", python)):
return join(prefix, "bin", python)
def stdlib_functions():
A = UnexpectedClass()
acos(A) # Noncompliant
B = datetime.tzinfo()
B.tzname(
42
) # Noncompliant {{Change this argument; Function "tzname" expects a different type}}
# ^^
select.select([], [], [], 0)
time.sleep(1) # OK
x = time.gmtime(int(time.time()))
x = time.gmtime(secs=int(time.time()))
time.sleep(True) # OK, converted to 1
time.sleep(1j) # FN, considered duck type compatible
genericpath.isfile("some/path")
genericpath.isfile(42) # Noncompliant
my_list = [1, 2, 3]
_heapq.heapify(
42
) # Noncompliant {{Change this argument; Function "heapify" expects a different type}}
# ^^
_heapq.heapify(my_list)
imap4 = imaplib.IMAP4()
imap4.setannotation(42) # FN, we do not handle variadic parameters
imap4.setannotation("some string") # OK
str_tuple = "string", "another string"
imap4.setannotation(str_tuple) # OK
def print_page():
if not _printer:
return "there is no printer", 400
if request.method == 'POST':
if request.files and 'uploadfile' in request.files:
file = request.files['uploadfile']
if file and beaglebone_helpers.allowed_file(file.filename):
filename = secure_filename(file.filename)
upload_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
try:
_logger.info("Saving file %s to %s", filename, upload_path)
file.save(upload_path)
except:
_logger.warn("unable to save file %s to %s", filename, upload_path)
elif 'printfile' in request.form:
filename = request.form['printfile']
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
if isfile(file_path) and beaglebone_helpers.allowed_file(filename):
_printer.prepared_file = file_path
_logger.info("Printing %s", _printer.prepared_file)
global _print_thread
_print_thread = GCodePrintThread(_printer.prepared_file, _printer, None)
_print_thread.start()
template_dictionary = templating_defaults()
files = [f for f in listdir(app.config['UPLOAD_FOLDER'])
if isfile(app.config['UPLOAD_FOLDER'] + "/" + f) and fnmatch.fnmatch(f, '*.gcode')]
#todo would like to http://stackoverflow.com/questions/6591931/getting-file-size-in-python
#http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
#http://stackoverflow.com/questions/237079/how-to-get-file-creation-modification-date-times-in-python
template_dictionary['files'] = files
if _printer.prepared_file:
template_dictionary['print_file'] = _printer.prepared_file.rsplit('/', 1)[1]
return render_template("print.html", **template_dictionary)
def _read_pickled(self, etag, mod_stamp):
if not isfile(self.cache_path_pickle) or not isfile(self.cache_path_json):
# Don't trust pickled data if there is no accompanying json data
return None
try:
if isfile(self.cache_path_pickle):
log.debug("found pickle file %s", self.cache_path_pickle)
with open(self.cache_path_pickle, 'rb') as fh:
_pickled_state = pickle.load(fh)
except Exception as e:
log.debug("Failed to load pickled repodata.", exc_info=True)
rm_rf(self.cache_path_pickle)
return None
def _check_pickled_valid():
yield _pickled_state.get('_url') == self.url_w_credentials
yield _pickled_state.get('_schannel') == self.channel.canonical_name
yield _pickled_state.get('_add_pip') == context.add_pip_as_python_dependency
yield _pickled_state.get('_mod') == mod_stamp
yield _pickled_state.get('_etag') == etag
yield _pickled_state.get('_pickle_version') == REPODATA_PICKLE_VERSION
if not all(_check_pickled_valid()):
log.debug("Pickle load validation failed for %s at %s.",
self.url_w_subdir, self.cache_path_json)
return None
return _pickled_state
def _read_pickled(self, etag, mod_stamp):
if not isfile(self.cache_path_pickle) or not isfile(
self.cache_path_json):
# Don't trust pickled data if there is no accompanying json data
return None
try:
if isfile(self.cache_path_pickle):
log.debug("found pickle file %s", self.cache_path_pickle)
with open(self.cache_path_pickle, 'rb') as fh:
_pickled_state = pickle.load(fh)
except Exception as e:
log.debug("Failed to load pickled repodata.", exc_info=True)
rm_rf(self.cache_path_pickle)
return None
def _check_pickled_valid():
yield _pickled_state.get('_url') == self.url_w_credentials
yield _pickled_state.get(
'_schannel') == self.channel.canonical_name
yield _pickled_state.get(
'_add_pip') == context.add_pip_as_python_dependency
yield _pickled_state.get('_mod') == mod_stamp
yield _pickled_state.get('_etag') == etag
yield _pickled_state.get(
'_pickle_version') == REPODATA_PICKLE_VERSION
if not all(_check_pickled_valid()):
log.debug("Pickle load validation failed for %s at %s.",
self.url_w_subdir, self.cache_path_json)
return None
return _pickled_state
def read_pickled_repodata(cache_path, channel_url, schannel, priority, etag,
mod_stamp):
pickle_path = get_pickle_path(cache_path)
# Don't trust pickled data if there is no accompanying json data
if not isfile(pickle_path) or not isfile(cache_path):
return None
try:
if isfile(pickle_path):
log.debug("found pickle file %s", pickle_path)
with open(pickle_path, 'rb') as f:
repodata = pickle.load(f)
except Exception as e:
import traceback
log.debug("Failed to load pickled repodata.\n%s",
traceback.format_exc())
rm_rf(pickle_path)
return None
def _check_pickled_valid():
yield repodata.get('_url') == channel_url
yield repodata.get('_schannel') == schannel
yield repodata.get('_add_pip') == context.add_pip_as_python_dependency
yield repodata.get('_mod') == mod_stamp
yield repodata.get('_etag') == etag
yield repodata.get('_pickle_version') == REPODATA_PICKLE_VERSION
if not all(_check_pickled_valid()):
return None
if int(repodata['_priority']) != priority:
log.debug("setting priority for %s to '%d'", repodata.get('_url'),
priority)
repodata['_priority']._priority = priority
return repodata
def read_pickled_repodata(cache_path, channel_url, schannel, priority, etag, mod_stamp):
pickle_path = get_pickle_path(cache_path)
# Don't trust pickled data if there is no accompanying json data
if not isfile(pickle_path) or not isfile(cache_path):
return None
try:
if isfile(pickle_path):
log.debug("found pickle file %s", pickle_path)
with open(pickle_path, 'rb') as f:
repodata = pickle.load(f)
except Exception as e:
import traceback
log.debug("Failed to load pickled repodata.\n%s", traceback.format_exc())
rm_rf(pickle_path)
return None
def _check_pickled_valid():
yield repodata.get('_url') == channel_url
yield repodata.get('_schannel') == schannel
yield repodata.get('_add_pip') == context.add_pip_as_python_dependency
yield repodata.get('_mod') == mod_stamp
yield repodata.get('_etag') == etag
yield repodata.get('_pickle_version') == REPODATA_PICKLE_VERSION
if not all(_check_pickled_valid()):
return None
if int(repodata['_priority']) != priority:
log.debug("setting priority for %s to '%d'", repodata.get('_url'), priority)
repodata['_priority']._priority = priority
return repodata
def _build(self, locale, messages, path, pathGlobal=None):
'''
Builds a catalog based on the provided locale paths, the path is used as the main source any messages that are not
found in path locale but are part of messages will attempt to be extracted from the global path locale.
@param locale: Locale
The locale.
@param messages: Iterable(Message)
The messages to build the PO file on.
@param path: string
The path of the targeted PO file from the locale repository.
@param pathGlobal: string|None
The path of the global PO file from the locale repository.
@return: file like object
File like object that contains the PO file content
'''
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
assert isinstance(messages, Iterable), 'Invalid messages %s' % messages
assert isinstance(path, str), 'Invalid path %s' % path
assert pathGlobal is None or isinstance(pathGlobal, str), 'Invalid global path %s' % pathGlobal
if isfile(path):
with open(path) as fObj: catalog = read_po(fObj, locale)
else:
catalog = Catalog(locale, creation_date=datetime.now(), **self.catalog_config)
if pathGlobal and isfile(pathGlobal):
with open(pathGlobal) as fObj: catalogGlobal = read_po(fObj, locale)
else:
catalogGlobal = None
self._processCatalog(catalog, messages, fallBack=catalogGlobal)
catalog.revision_date = datetime.now()
return catalog
def test_isfile(self):
self.assertIs(genericpath.isfile(support.TESTFN), False)
f = open(support.TESTFN, "wb")
try:
f.write(b"foo")
f.close()
self.assertIs(genericpath.isfile(support.TESTFN), True)
os.remove(support.TESTFN)
os.mkdir(support.TESTFN)
self.assertIs(genericpath.isfile(support.TESTFN), False)
os.rmdir(support.TESTFN)
finally:
if not f.close():
f.close()
try:
os.remove(support.TESTFN)
except os.error:
pass
try:
os.rmdir(support.TESTFN)
except os.error:
pass
self.assertRaises(TypeError, genericpath.isdir)
def test_samefile(self):
f = open(support.TESTFN + "1", "wb")
try:
f.write(b"foo")
f.close()
self.assertIs(
genericpath.samefile(support.TESTFN + "1",
support.TESTFN + "1"), True)
# If we don't have links, assume that os.stat doesn't return resonable
# inode information and thus, that samefile() doesn't work
if hasattr(os, "symlink"):
os.symlink(support.TESTFN + "1", support.TESTFN + "2")
self.assertIs(
genericpath.samefile(support.TESTFN + "1",
support.TESTFN + "2"), True)
os.remove(support.TESTFN + "2")
f = open(support.TESTFN + "2", "wb")
f.write(b"bar")
f.close()
self.assertIs(
genericpath.samefile(support.TESTFN + "1",
support.TESTFN + "2"), False)
finally:
if not f.close():
f.close()
try:
os.remove(support.TESTFN + "1")
except os.error:
pass
try:
os.remove(support.TESTFN + "2")
except os.error:
pass
self.assertRaises(TypeError, genericpath.samefile)
def run_annotators(cnf, vcf_fpath):
work_dir = cnf['work_dir']
annotated = False
if 'gatk' in cnf:
annotated = True
vcf_fpath = _gatk(cnf, vcf_fpath, cnf.get('bam'), work_dir)
if 'dbsnp' in cnf:
annotated = True
vcf_fpath = _snpsift_annotate(cnf, cnf['dbsnp'],
'dbsnp', vcf_fpath, work_dir)
if 'cosmic' in cnf:
annotated = True
vcf_fpath = _snpsift_annotate(cnf, cnf['cosmic'],
'cosmic', vcf_fpath, work_dir)
if 'custom_vcfs' in cnf:
for dbname, custom_conf in cnf['custom_vcfs'].items():
annotated = True
vcf_fpath = _snpsift_annotate(
cnf, custom_conf, dbname, vcf_fpath, work_dir)
if 'dbnsfp' in cnf:
annotated = True
vcf_fpath = _snpsift_db_nsfp(cnf, vcf_fpath, work_dir)
if 'snpeff' in cnf:
annotated = True
_remove_annotation(cnf, 'EFF', vcf_fpath, work_dir)
vcf_fpath, summary_fpath, genes_fpath = _snpeff(cnf, vcf_fpath, work_dir)
if isfile(join(cnf['output_dir'], summary_fpath)):
os.remove(join(cnf['output_dir'], summary_fpath))
if isfile(join(cnf['output_dir'], genes_fpath)):
os.remove(join(cnf['output_dir'], genes_fpath))
if file_exists(summary_fpath):
shutil.move(summary_fpath, cnf['output_dir'])
if file_exists(genes_fpath):
shutil.move(genes_fpath, cnf['output_dir'])
if cnf.get('tracks'):
for track in cnf['tracks']:
annotated = True
vcf_fpath = _tracks(cnf, track, vcf_fpath, work_dir)
if annotated:
vcf_fpath = _filter_fields(cnf, vcf_fpath, work_dir)
# Copying final VCF
final_vcf_fname = add_suffix(basename(cnf['vcf']), 'anno')
final_vcf_fpath = join(cnf['output_dir'], final_vcf_fname)
if isfile(final_vcf_fpath):
os.remove(final_vcf_fpath)
shutil.copyfile(vcf_fpath, final_vcf_fpath)
return final_vcf_fpath
else:
info('No annotations were run on ' + vcf_fpath + '. Please, specify some in run_info.')
return None
def ida_extract(path, args, procedure_name):
"""
returns work_path where the results were stored
"""
# Run IDA headless and extract the procedures
extract_script_path = path_join(dirname(realpath(__file__)), "py2",
'bin_extract.py')
extract_command = 'cd {};TVHEADLESS=1 {} -B -S"{}{}" {} > {}'. \
format(dirname(path), args['idal64_path'], extract_script_path,
"" if procedure_name is None else " " + procedure_name, basename(path), FNULL.name)
r, output = getstatusoutput(extract_command)
if r != 0:
logging.warning(
"IDA command {} had errors. Giving it another (last) chance.".
format(extract_command))
r, output = getstatusoutput(extract_command)
if r != 0:
raise Exception(
"IDA command {} returned error code {}, even after retry!".
format(extract_command, r))
work_path = dirname(path)
# float ida critical errors
ida_log_file_path = path_join(work_path, log_file_name)
if isfile(ida_log_file_path):
with open(ida_log_file_path) as ida_log_file:
for line in ida_log_file.readlines():
if line.startswith("CRITICAL"):
logging.critical(
"file={}, IDA Critical Message - {}".format(
path, line))
raise RuntimeError(
"Error in IDA run for - {}".format(path))
else:
logging.warning("IDA log file not found! ({})".format(path))
if not args['keep_temps']:
for temporary in [
path_join(dirname(path), f) for f in listdir(dirname(path))
]:
if temporary.endswith(
IndexedExeFile.get_filename()) or temporary.endswith(
IndexedProcedure.get_filename_suffix()):
continue
if temporary.endswith("i64") or temporary == path:
continue
if temporary.endswith(extern_dump_file_name):
continue
# ida will only make tmp files no dirs, so we can do this..
if isfile(temporary):
remove(temporary)
return work_path
def bgzip_and_tabix(cnf, vcf_fpath, tabix_parameters='', **kwargs):
gzipped_fpath = join(vcf_fpath + '.gz')
tbi_fpath = gzipped_fpath + '.tbi'
if cnf.reuse_intermediate and \
file_exists(gzipped_fpath) and \
file_exists(tbi_fpath) and getctime(tbi_fpath) >= getctime(gzipped_fpath):
info('Actual compressed VCF and index exist, reusing')
return gzipped_fpath
info('Compressing and tabixing VCF file, writing ' + gzipped_fpath + '(.tbi)')
bgzip = get_system_path(cnf, 'bgzip')
tabix = get_system_path(cnf, 'tabix')
if not bgzip:
err('Cannot index VCF because bgzip is not found in PATH or ' + cnf.sys_cnf)
if not tabix:
err('Cannot index VCF because tabix is not found in PATH or ' + cnf.sys_cnf)
if not bgzip and not tabix:
return vcf_fpath
retrying = False
while True:
if isfile(tbi_fpath): os.remove(tbi_fpath)
if isfile(vcf_fpath):
if isfile(gzipped_fpath):
os.remove(gzipped_fpath)
info('BGzipping VCF')
cmdline = '{bgzip} {vcf_fpath}'.format(**locals())
call(cnf, cmdline, None, **kwargs)
else:
if not verify_file(gzipped_fpath):
err('Neither uncompressed ' + vcf_fpath + ' nor ' + gzipped_fpath + ' exist')
return None
info('Tabixing VCF')
cmdline = '{tabix} {tabix_parameters} {gzipped_fpath}'.format(**locals())
exit_on_error = False
if retrying:
exit_on_error = True
kwargs['exit_on_error'] = exit_on_error
call(cnf, cmdline, **kwargs)
if isfile(gzipped_fpath + '.tbi'):
break
if retrying:
critical('Cannot tabix ' + vcf_fpath)
if not isfile(vcf_fpath):
call(cnf, 'gunzip ' + gzipped_fpath, None)
retrying = True
return gzipped_fpath
def in_venv():
"""
:return: True if in running from a virtualenv
Has to detect the case where the python binary is run
directly, so VIRTUAL_ENV may not be set
"""
global _in_venv
logger.debug("in_venv enter with interpreter %s", sys.executable)
if _in_venv is not None:
logger.debug("in_venv return cached value %s", _in_venv)
return _in_venv
if not (os.path.isfile(ORIG_PREFIX_TXT) or os.path.isfile(PY_VENV_CFG)):
logger.debug("in_venv no orig_prefix_txt [%s]", ORIG_PREFIX_TXT)
logger.debug("in_venv no py_venv_cfg [%s]", PY_VENV_CFG)
# TODO - check this is actually valid !
_in_venv = False
return _in_venv
if 'VIRTUAL_ENV' in os.environ:
logger.debug("in_venv VIRTUAL_ENV set.")
_in_venv = True
else:
# Find first python in path ... if its not this one,
# ...we are in a different python environment
python = basename(sys.executable)
for p in os.environ['PATH'].split(os.pathsep):
py_path = join(p, python)
if isfile(py_path):
_in_venv = sys.executable != py_path
logger.debug("in_venv: [%s], sys.executable is not first python in path [%s]", _in_venv, py_path)
break
return _in_venv
def select_game():
GAMES_DIR = 'games'
DEFAULT_CONF_FILE = 'conf_file'
games = [x for x in os.listdir(GAMES_DIR) if
isdir(GAMES_DIR + "/" + x) and isfile(GAMES_DIR + "/" + x + "/" + DEFAULT_CONF_FILE)]
if len(games) < 1:
print("No games are present in the default directory (Do you have put them in the folder \'games\'?)")
return False
while True:
print("Available games:")
for i, game in enumerate(games):
print("\t" + str(i) + ") " + str(game))
try:
choosed = int(input('Select the puzzle that you want to solve: '))
print("You have chosen: " + str(games[choosed]))
print()
print("--------------------------------")
try:
return GAMES_DIR + os.sep + games[choosed] + os.sep, DEFAULT_CONF_FILE
except Exception as ex:
print('An error occurs :(')
print("--------------------------------")
print()
except (ValueError, TypeError):
return None, None
def adjust_proteomes(prot_fpaths, proteomes_dir, prot_id_field):
if not isdir(proteomes_dir):
mkdir(proteomes_dir)
for proteome_fpath in prot_fpaths:
prot_ids = set()
records = []
taxon_id, ext = splitext(basename(proteome_fpath))
ext = '.fasta'
for seq in SeqIO.parse(proteome_fpath, 'fasta'):
fields = seq.id.replace('|', ' ').split()
if len(fields) > prot_id_field:
prot_id = fields[prot_id_field]
elif len(fields) > 0:
prot_id = fields[-1]
else:
log.error('Incorrect fasta id: ' + str(seq.id))
return 1
if len(prot_id) > 30:
prot_id = prot_id[:17] + '...' + prot_id[-10:]
if prot_id in prot_ids:
log.error('Fasta {proteome_fpath} contains duplicate id: '
'{prot_id}'.format(**locals()))
return 1
prot_ids.add(prot_id)
seq.id = taxon_id + '|' + prot_id
records.append(seq)
out_fpath = join(proteomes_dir, taxon_id + ext)
if isfile(out_fpath):
remove(out_fpath)
SeqIO.write(records, out_fpath, 'fasta')
return 0
def resample_allfiles(
results_dir: Optional[str] = None, resampled_dir: Optional[str] = None
) -> int:
"""Resample all files inside results_dir folder.
Parameters
----------
results_dir: str
Directory containing results to resample.
resampled_dir: str
Directory to save resampled results.
"""
if results_dir is None:
results_dir = eniric.paths["results"] # type: ignore
if resampled_dir is None:
resampled_dir = eniric.paths["resampled"] # type: ignore
# Getting a list of all the files
onlyfiles = [f for f in os.listdir(results_dir) if isfile(join(results_dir, f))]
for spectrum_file in onlyfiles:
if spectrum_file.endswith(".dat"):
resampler(
spectrum_file, results_dir=results_dir, resampled_dir=resampled_dir
)
return 0
def run_manifests(stack_dir):
images = read_var('IMAGES_BUILT')
image_deps = json2series(read_dict('image-dependency.json'), 'dep',
'image')
store_series(image_deps, 'image-dependency')
# Write image dependency table to wiki
dep_table_fp = 'wiki/Image Dependency.md'
if isfile(dep_table_fp):
old_csv = strip_csv_from_md(dep_table_fp)
csv_concat(old_csv, 'artifacts/image-dependency.csv',
'artifacts/image-dependency-updated.csv')
csv_embed_markdown('artifacts/image-dependency-updated.csv',
dep_table_fp, 'Image Dependency')
else:
csv_embed_markdown('artifacts/image-dependency.csv', dep_table_fp,
'Image Dependency')
specs = get_specs(path.join(stack_dir, 'spec.yml'))
for image in images:
keys = list(filter(lambda x: x in image, specs['images']))
assert len(keys) == 1
image_key = keys[0]
print('Running image manifest for', image)
run_report(specs, image_key, image=image)
insert_history('wiki/Home.md')
def _show_dir(path):
html = ''
for item in os.listdir(settings.TINYMCE_IMAGES_STORAGE_ROOT + path):
if isfile(settings.TINYMCE_IMAGES_STORAGE_ROOT + path + item):
file = ImageFile(settings.TINYMCE_IMAGES_STORAGE_ROOT, path + item)
html += render_to_string('tinymce_images_filelist.html', {'image' : file})
return html
def read_data_from_csv(csv_path, sort=True):
"""
Read data from CSV into DataFrame.
:param csv_path: absolute path to CSV file
:param sort: whether to sort the files by file size
:return:
df: pd.DataFrame containing the CSV data
duration: integer representing the total length of the audio by summing over the 'wav_length' column
from the CSV. If such a column is not present, the value returned is math.inf
"""
if not isfile(csv_path):
print(f'ERROR: CSV file {csv_path} does not exist!', file=sys.stderr)
exit(0)
print(f'Reading samples from {csv_path}...', end='')
df = pd.read_csv(csv_path, sep=',', encoding='utf-8')
print(f'done! ({len(df.index)} samples', end='')
duration = math.inf
if 'wav_length' in df:
duration = df['wav_length'].sum()
avg_duration = df['wav_length'].mean()
print(
f', {timedelta(seconds=duration)}, Ø audio length: {avg_duration:.4f} seconds)'
)
else:
print(')')
if sort:
df = df.sort_values(by='wav_filesize', ascending=True)
avg_trans_length = np.mean([len(trans) for trans in df['transcript']])
print(f'average transcript length: {avg_trans_length}')
return df.reset_index(drop=True), duration
def in_venv():
"""
:return: True if in running from a virtualenv
Has to detect the case where the python binary is run
directly, so VIRTUAL_ENV may not be set
"""
global _in_venv
if _in_venv is not None:
return _in_venv
if not os.path.isfile(orig_prefix):
logger.debug("in_venv no orig_prefix [%s]", orig_prefix)
# TODO - check this is actually valid !
_in_venv = False
return _in_venv
if 'VIRTUAL_ENV' in os.environ:
logger.debug("in_venv VIRTUAL_ENV set.")
_in_venv = True
else:
# Find first python in path ... if its not this one,
# ...we are in a different python
python = basename(sys.executable)
for p in os.environ['PATH'].split(os.pathsep):
py_path = join(p, python)
if isfile(py_path):
logger.debug("in_venv py_at [%s] return: %s", (py_path, sys.executable != py_path))
_in_venv = sys.executable != py_path
break
return _in_venv
def read(self):
for node in os.listdir(join(self.root, self.path)):
full_path = join(self.path, node)
if isdir(join(self.root, full_path)):
self.folders.append(Folder(self.root, full_path))
if isfile(join(self.root, full_path)):
self.files.append(File(self.root, full_path))
def adjust_proteomes(prot_fpaths, proteomes_dir, prot_id_field):
if not isdir(proteomes_dir):
mkdir(proteomes_dir)
for proteome_fpath in prot_fpaths:
prot_ids = set()
records = []
taxon_id, ext = splitext(basename(proteome_fpath))
ext = '.fasta'
for seq in SeqIO.parse(proteome_fpath, 'fasta'):
fields = seq.id.replace('|', ' ').split()
if len(fields) > prot_id_field:
prot_id = fields[prot_id_field]
elif len(fields) > 0:
prot_id = fields[-1]
else:
log.error('Incorrect fasta id: ' + str(seq.id))
return 1
if len(prot_id) > 30:
prot_id = prot_id[:17] + '...' + prot_id[-10:]
if prot_id in prot_ids:
log.error('Fasta {proteome_fpath} contains duplicate id: '
'{prot_id}'.format(**locals()))
return 1
prot_ids.add(prot_id)
seq.id = taxon_id + '|' + prot_id
records.append(seq)
out_fpath = join(proteomes_dir, taxon_id + ext)
if isfile(out_fpath):
remove(out_fpath)
SeqIO.write(records, out_fpath, 'fasta')
return 0
def match_or_trust(self, host, der_encoded_certificate):
base64_encoded_certificate = b64encode(der_encoded_certificate)
if isfile(self.path):
with open(self.path) as f_in:
for line in f_in:
known_host, _, known_cert = line.strip().partition(":")
known_cert = known_cert.encode("utf-8")
if host == known_host:
return base64_encoded_certificate == known_cert
# First use (no hosts match)
try:
makedirs(dirname(self.path))
except OSError:
pass
f_out = os_open(self.path, O_CREAT | O_APPEND | O_WRONLY,
0o600) # TODO: Windows
if isinstance(host, bytes):
os_write(f_out, host)
else:
os_write(f_out, host.encode("utf-8"))
os_write(f_out, b":")
os_write(f_out, base64_encoded_certificate)
os_write(f_out, b"\n")
os_close(f_out)
return True
def tenant_bills_to_docx(self):
""" Compose a single bill for a tenant_id, save it to disk.
returns a string containing the path to the saved file. """
for t in self.rtp.tenantBillList:
if not self.rtp.bill_tenant_0:
if t.tenant_id == 0:
continue
t_bill = MailMerge(self.rtp.pdf_docx_template)
t_bill.merge(
name=format_values(t.tenant_name),
month_year=format_values("{} {}".format(
self.rtp.month_str, self.rtp.year)),
total=format_values(t.charge_total),
date=format_values(
datetime.datetime.now().strftime('%Y %B %d')),
room_rate=format_values(t.charge_room),
gas=format_values(t.charge_gas),
internet=format_values(t.charge_internet),
electricity=format_values(t.charge_electricity),
recurring=format_values(t.charge_recurring),
other=format_values(t.charge_other),
memo_gas=format_values(t.memo_gas),
memo_internet=format_values(t.memo_internet),
memo_electricity=format_values(t.memo_electricity),
memo_recurring=format_values(t.memo_other),
memo_other=format_values(t.memo_other),
)
docx = self.rtp.google_path + "bills\\" + self.__format_file_name(
t, "docx")
if isfile(docx):
remove(docx)
t_bill.write(docx)
t.docx_long_name = docx
self.logger.debug("Created docx_long_name file: {}".format(docx))
def addpackage(sitedir, pthfile, known_dirs=None):
"""
Wrapper for site.addpackage
Try and work out which directories are added by
the .pth and add them to the known_dirs set
"""
known_dirs = set(known_dirs or [])
with open(join(sitedir, pthfile)) as f:
for n, line in enumerate(f):
if line.startswith("#"):
continue
line = line.rstrip()
if line:
if line.startswith(("import ", "import\t")):
exec (line, globals(), locals())
continue
else:
p_rel = join(sitedir, line)
p_abs = abspath(line)
if isdir(p_rel):
os.environ['PATH'] += env_t(os.pathsep + p_rel)
sys.path.append(p_rel)
added_dirs.add(p_rel)
elif isdir(p_abs):
os.environ['PATH'] += env_t(os.pathsep + p_abs)
sys.path.append(p_abs)
added_dirs.add(p_abs)
if isfile(pthfile):
site.addpackage(sitedir, pthfile, known_dirs)
else:
logging.debug("pth file '%s' not found")
def _lastModified(self, locale, component=None, plugin=None):
'''
Provides the last modification time stamp for the provided locale. You can specify the component id in order to
get the last modification for the component domain, or plugin or either to get the global domain modification.
@param locale: Locale
The locale to get the last modification for.
@param component: string|None
The component id to get the last modification for.
@param plugin: string|None
The plugin id to get the last modification for.
@return: datetime|None
The last modification time stamp, None if there is no such time stamp available.
'''
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
assert not(component and plugin), 'Cannot process a component id %s and a plugin id %s' % (component, plugin)
q = QSource()
q.lastModified.orderDesc()
if component: q.component = component
elif plugin: q.plugin = plugin
sources = self.sourceService.getAll(0, 1, q=q)
try: lastModified = next(iter(sources)).LastModified
except StopIteration: lastModified = None
path = self._filePath(locale, component, plugin)
if isfile(path):
lastModified = max(lastModified, datetime.fromtimestamp(os.stat(path).st_mtime))
return lastModified
def add_env_path(path):
# Add package folder to searching path
# Search the dir that contains file '__dbreport__.py'
for dirname in work_all_path(path):
if isfile(dirname + os.sep +
'__dbreport__.py') and dirname not in sys.path:
sys.path.append(dirname)
def _lastModified(self, locale, component=None, plugin=None):
'''
Provides the last modification time stamp for the provided locale. You can specify the component id in order to
get the last modification for the component domain, or plugin or either to get the global domain modification.
@param locale: Locale
The locale to get the last modification for.
@param component: string|None
The component id to get the last modification for.
@param plugin: string|None
The plugin id to get the last modification for.
@return: datetime|None
The last modification time stamp, None if there is no such time stamp available.
'''
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
assert not (
component and plugin
), 'Cannot process a component id %s and a plugin id %s' % (component,
plugin)
q = QSource()
q.lastModified.orderDesc()
if component: q.component = component
elif plugin: q.plugin = plugin
sources = self.sourceService.getAll(0, 1, q=q)
try:
lastModified = next(iter(sources)).LastModified
except StopIteration:
lastModified = None
path = self._filePath(locale, component, plugin)
if isfile(path):
lastModified = max(lastModified,
datetime.fromtimestamp(os.stat(path).st_mtime))
return lastModified
def read(self):
for node in os.listdir(join(self.root, self.path)):
full_path = join(self.path, node)
if isdir(join(self.root, full_path)):
self.folders.append(Folder(self.root, full_path))
if isfile(join(self.root, full_path)):
self.files.append(File(self.root, full_path))
def main(argv1, argv2):
# load necessary configuration
# argv1 -- output file path
# argv2 -- corpus path
p_model = load_model(
"/opt/gsda/EUPEG/Geoparsers/camCoder/data/weights"
) # weights to be downloaded from Cambridge Uni repo, see GitHub.
p_word_to_index = cPickle.load(
open(u"/opt/gsda/EUPEG/Geoparsers/camCoder/codes/data/words2index.pkl")
) # This is the vocabulary file
# Example usage of the geoparse function below reading from a directory and parsing all files.
directory = unicode(str(argv2), 'utf-8')
files = [f for f in listdir(directory) if isfile(directory + f)]
# output_file = open(argv1, 'w+')
with open(argv1, "w") as wf:
for i in range(0, len(files)):
# print i
full_line = ""
for line in codecs.open(directory + str(i),
encoding="utf-8",
errors='ignore'):
full_line = full_line + line
article_parsing = geoparse(full_line, "", p_model, p_word_to_index)
# write result into the output
wf.write(article_parsing)
wf.write('\n')
def __init__(self):
self.cache = "parscit_cache"
if "PARSCIT" not in environ:
raise ValueError("Enviroment variable PARSCIT must point to PARSCIT source")
self.script = join(environ["PARSCIT"], "bin", "citeExtract.pl")
if not isfile(self.script):
raise ValueError()
def migrate_copy2_remote():
result_file = open("/home/work/data/collie/local_modules/results.log",
"wb+")
webhdfs = WebHdfs()
for version_id in range(0, 2000):
try:
base_path = "/home/work/data/collie/local_modules/" + str(
version_id) + "/"
files = [
f for f in listdir(base_path) if isfile(join(base_path, f))
]
if len(files) <= 0:
continue
file_name = files[0]
version_file = open(join(base_path, file_name), 'rb')
checksum = checksum_file(version_file)
file_name_new = file_name + "_" + checksum
os.rename(join(base_path, file_name), join(base_path,
file_name_new))
remote_path = "/user/h_sns/collie/modules/"
webhdfs.copyFromLocal(join(base_path, file_name_new), remote_path)
result_file.write(
str(version_id) + "\t" + remote_path + file_name_new + "\n")
print "%d is saving to remote %s" % (version_id,
remote_path + file_name_new)
except IOError:
continue
except OSError:
continue
def __init__(self, logFile='log.txt'):
self.logFile = logFile
self.sightingFile = None
self.ariesFile = None
self.starFile = None
self.sightingError = 0
self.sighting_tuples = []
if not isinstance(logFile, str):
raise ValueError('Fix.__init__: "logFile" is a string')
if (len(logFile) < 1):
raise ValueError(
'Fix.__init__: "logFile" should have a length .GE. 1')
if not isfile(logFile):
try:
log = open(logFile, 'w')
except:
raise ValueError('Fix.__init__: "logFile" can not be created')
else:
try:
log = open(logFile, 'a')
except:
raise ValueError(
'Fix.__init__: "logFile" can not be opened for appending')
logEntry = self.message("Log file:\t" + os.path.abspath(self.logFile))
try:
log.write(logEntry)
except:
raise ValueError('Fix.__init__: "logFile" can not be appended')
log.close()
def catch_all(path):
current_app.logger.debug("Working in: %s",
os.environ.get("SERVER_STATIC_FOLDER", ""))
current_app.logger.debug("FALLBACK for path: %s", path)
# check compression
compress = os.environ["COMPRESS"].lower() == "true"
if compress and (path.endswith(".js") or path.endswith(".css")):
current_app.logger.debug("Use compression")
path += ".gz"
else:
compress = False
# detect language from url path (en|hu)
languages = os.environ["LANGUAGES"].split(" ")
result = re.search("(" + "|".join(languages) + ")", path)
language = result.group(0) if result else ""
current_app.logger.debug("Language: %s from %s", language
or "No language in URL", languages)
if language == "en":
path = path.replace("en/", "")
current_app.logger.debug("FALLBACK for path processed: %s", path)
# return with file if exists
current_app.logger.debug("Checking for %s", path)
if isfile(join(current_app.config["WEBAPP_SOURCE"], path)):
current_app.logger.debug("Path exists without language: %s", path)
response = send_from_directory(current_app.config["WEBAPP_SOURCE"],
path)
if compress:
response.headers["Content-Encoding"] = "gzip"
return response
elif language and isfile(
join(current_app.config["WEBAPP_SOURCE"], language, "index.html")):
current_app.logger.debug("Path exists with language: %s",
join(language, "index.html"))
return send_from_directory(
join(current_app.config["WEBAPP_SOURCE"], language), "index.html")
# or return with the index file
current_app.logger.debug("INDEX without language: %s",
join(language, "index.html"))
return send_from_directory(current_app.config["WEBAPP_SOURCE"],
"index.html")
def initialize():
if isfile(config.case_db_path):
if config.reinitialize_case_db_on_startup:
case_db.tearDown()
remove(config.case_db_path)
case_db.create()
else:
case_db.create()
def __init__(self):
self.cache = "parscit_cache"
if "PARSCIT" not in environ:
raise ValueError(
"Enviroment variable PARSCIT must point to PARSCIT source")
self.script = join(environ["PARSCIT"], "bin", "citeExtract.pl")
if not isfile(self.script):
raise ValueError()
def getfiles(self, src_dir):
files = []
for f in os.listdir(src_dir):
if isfile(os.path.join(src_dir, f)):
if not f.startswith('.'):
files.append(os.path.join(src_dir, f))
return files
def get():
if exists(VersionManager.__location) and isfile(VersionManager.__location):
try:
with open(VersionManager.__location) as f:
return json.load(f)
except:
LOG.error("Failed to load version from '%s'" % VersionManager.__location)
return {"coreVersion": None, "enclosureVersion": None}
def __load(config, location):
if exists(location) and isfile(location):
try:
config.update(load_commented_json(location))
LOG.debug("Configuration '%s' loaded" % location)
except Exception, e:
LOG.error("Error loading configuration '%s'" % location)
LOG.error(repr(e))
def __load(config, location):
if exists(location) and isfile(location):
try:
with open(location) as f:
config.update(json.load(f))
LOG.debug("Configuration '%s' loaded" % location)
except Exception, e:
LOG.error("Error loading configuration '%s'" % location)
LOG.error(repr(e))
def index_image_folder(folder_path):
for f in listdir(folder_path):
path = join(folder_path, f)
if isfile(path):
is_image = any(path.endswith("." + ext) for ext in config.IMAGE_EXTENSIONS)
if is_image: # check extension, only index images
index_image(folder_path=folder_path, file_name=f)
elif isdir(path):
index_image_folder(path)
def __load(config, location):
if exists(location) and isfile(location):
try:
cobj = ConfigObj(location)
config = ConfigurationLoader.__merge(config, cobj)
logger.debug("Configuration '%s' loaded" % location)
except Exception, e:
logger.error("Error loading configuration '%s'" % location)
logger.error(repr(e))
def process_item(self, item, spider):
if 'prices' in spider.name:
self._register(item)
if isfile(self._csv_file):
self._write()
else:
self._initialize()
self.process_item(item, spider)
return item
def __init__(self):
my_cache_file_path = LibExpRepo.my_cache
if not isfile(my_cache_file_path) and isfile(my_cache_file_path +
".zip"):
import zipfile
logging.warning("Extracting LIBEXP zip file")
with zipfile.ZipFile(my_cache_file_path + ".zip", 'r') as zip_ref:
zip_ref.extractall(".")
if isfile(my_cache_file_path):
with open(my_cache_file_path, "rb") as cache_file_in:
self._repo = pickle_load(cache_file_in)
logging.warning(
"Loaded LIBEXP from cache, (has ={} records)".format(
len(list(self._repo.keys()))))
return
def __load(config, location):
if exists(location) and isfile(location):
try:
with open(location) as f:
config.update(json.load(f))
LOG.debug("Configuration '%s' loaded" % location)
except Exception, e:
LOG.error("Error loading configuration '%s'" % location)
LOG.error(repr(e))
def check_hdf5(self, reload=False):
if reload:
Hdf5Creator().create_hdf5()
elif isfile(self.hdf5_path):
if not PytablesHdf5Manager(
self.hdf5_path).check_exist_dataframes(DS_ARRAY):
Hdf5Creator().create_hdf5()
else:
Hdf5Creator().create_hdf5()
def __load(config, location):
if exists(location) and isfile(location):
try:
cobj = ConfigObj(location)
config = ConfigurationLoader.__merge(config, cobj)
logger.debug("Configuration '%s' loaded" % location)
except Exception, e:
logger.error("Error loading configuration '%s'" % location)
logger.error(repr(e))
def find_project_data(p, levels_to_project_traits):
p = split(abspath(p))
if not p[1]:
return '', levels_to_project_traits
if isfile(join(p[0], 'data', database)):
return p[0], levels_to_project_traits
else:
return find_project_data(p[0], levels_to_project_traits+1)
def test_settings_nonworld():
"""
Check that settings_local.py exists, is somewhat hidden and has a secret key and database settings.
"""
pth = join(settings.BASE_DIR, 'source', 'local.py')
assert isfile(pth), 'you should have a local settings file with secret settings, located at "{0:s}"'.format(pth)
mode = stat(pth).st_mode
assert not mode & S_IROTH, 'local settings should not be world-readable (file "{0:s}")'.format(pth)
assert not mode & S_IWOTH, 'local settings should not be world-writable (file "{0:s}")'.format(pth)
assert not mode & S_IXOTH, 'local settings should not be world-executable (file "{0:s}")'.format(pth)
def get():
if (exists(VersionManager.__location) and
isfile(VersionManager.__location)):
try:
with open(VersionManager.__location) as f:
return json.load(f)
except:
LOG.error("Failed to load version from '%s'"
% VersionManager.__location)
return {"coreVersion": None, "enclosureVersion": None}
def get():
data_dir = expanduser(Configuration.get()['data_dir'])
version_file = join(data_dir, 'version.json')
if exists(version_file) and isfile(version_file):
try:
with open(version_file) as f:
return json.load(f)
except Exception:
LOG.error("Failed to load version from '%s'" % version_file)
return {"coreVersion": None, "enclosureVersion": None}
def check_common_args(params):
if params.start_from == 'uselog':
if not isfile(join(params.out_dir, config.log_fname)):
arg_parse_error(
'No %s in %s. Either check your path, or '
'change the --start-from option' %
(config.log_fname, params.out_dir))
if params.start_from:
params.overwrite = True
def getFilesFromFolder(infiles):
"""
:param infiles:
:return:
"""
only_files = [f for f in listdir(infiles) if isfile(join(infiles, f))]
if '.DS_Store' in only_files:
only_files.__delitem__(only_files.index('.DS_Store'))
return only_files
def classpath_string(self, scope):
resolved_classpath_file = os.path.join(self.package_cache, "%s-%s-%s" % (self.group_id, self.artifact_id, self.version))
if not isfile(resolved_classpath_file):
print "Resolving ivy dependency %s" % self
subprocess.check_call(["java", "-jar", self.ivy_executable, "-dependency",
self.group_id, self.artifact_id, self.version,
"-cachepath", resolved_classpath_file])
with open(resolved_classpath_file) as f:
return f.read().replace("\n", "")
def read(self):
for entry in os.listdir(join(self.root, self.path)):
full_path = join(self.path, entry)
if isdir(join(self.root, full_path)):
node = Folder(self.root, full_path)
self.folders.append(node)
if isfile(join(self.root, full_path)):
node = File(self.root, full_path)
self.files.append(node)
self.nodes.append(node)
self.nodes.sort(key=lambda node: node.key())
def __init__(self, dependent_id, dependency_id):
super(IvyDependencyJob, self).__init__(dependent_id, {})
self.ivy_executable = os.path.expanduser("~/.bacon.d/ivy.jar")
split_dep = dependency_id.split(":")
self.group_id = split_dep[0]
self.artifact_id = split_dep[1]
self.version = split_dep[2]
if not isfile(self.ivy_executable):
print "Downloading ivy..."
urllib.urlretrieve("http://central.maven.org/maven2/org/apache/ivy/ivy/2.3.0/ivy-2.3.0.jar", self.ivy_executable)
def get_spiders_files(spiders_directory=None):
"""
Return list of filename corresponding to JobSpider
:param spiders_directory: Path where search JobSpiders
:return: list of filename
"""
if spiders_directory is None:
spiders_directory = dirname(__file__) + '/spiders/'
return [file for file in glob.glob(spiders_directory + "/*.py")
if isfile(file)
and not file.endswith('__init__.py')]
def process_transcoding((flac_file, flac_root_path, mp3_target_path)):
try:
target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file)
if not isdir(dirname(target_mp3_file)):
try:
os.makedirs(dirname(target_mp3_file))
except OSError:
pass # other thread might have been faster
if isfile(target_mp3_file) and tags_are_equals(flac_file, target_mp3_file):
LOGGER.info('skipping %r as target mp3 file exists and seems to have the same tags', flac_file)
else:
transcode(flac_file, target_mp3_file)
except Exception as e:
LOGGER.exception('error during the transcoding of %r : %s' % (flac_file, e))
def compress(source_dir, dest_file):
relroot = abspath(join(source_dir, ".."))
with ZipFile(dest_file, 'w', compression=ZIP_DEFLATED) as zf:
for root, dirs, files in walk(source_dir):
zf.write(root, relpath(root, relroot))
for f in files:
fname = join(root, f)
if isfile(fname):
arcname = join(relpath(root, relroot), f)
#if '.cpan' not in fname and 'perl_modules' not in fname:
print ' ' + arcname # join(relpath(root, directory), f)
zf.write(fname, arcname)
