def get_subtitle(url, path):
in_data = urllib2.urlopen(url)
temp_file = NamedTemporaryFile()
temp_file.write(in_data.read())
in_data.close()
temp_file.seek(0)
if is_zipfile(temp_file.name):
zip_file = ZipFile(temp_file)
for name in zip_file.namelist():
# don't unzip stub __MACOSX folders
if '.srt' in name and '__MACOSX' not in name:
logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)]))
zip_file.extract(name, os.path.dirname(path))
zip_file.close()
elif is_rarfile(temp_file.name):
rar_path = path + '.rar'
logger.info('Saving rared subtitle as %s' % rar_path)
with open(rar_path, 'w') as out_file:
out_file.write(temp_file.read())
try:
import subprocess
#extract all .srt in the rared file
ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path])
if ret_code == 0:
logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path))
os.remove(rar_path)
except OSError:
logger.info('Unpacking rared subtitle failed.'
'Please, install unrar to automate this step.')
temp_file.close()
def _get_tmp_file(self):
''' Creates a tmp file with the file data '''
data = self.request.files['xml_file'][0]['body']
tmp_file = NamedTemporaryFile(delete=False)
tmp_file.write(data)
tmp_file.close()
return tmp_file.name
def main():
logging.basicConfig(format='%(message)s', level=logging.INFO)
args = parse_args()
from_date = args.from_date
to_date = args.to_date
s3_client = S3Client.connect_with_profile(args.aws_profile)
s3_bucket = args.s3_bucket
s3_key = "{}_{}.logs.json".format(from_date.isoformat(), to_date.isoformat())
sumo_access_id = args.sumo_access_id or os.environ["SUMOLOGIC_ACCESS_ID"]
sumo_access_key = args.sumo_access_key or os.environ["SUMOLOGIC_ACCESS_KEY"]
sumo_logic_client = SumoLogicClient(sumo_access_id, sumo_access_key)
logging.info(
"Searching for request logs in {} in range {} to {}".format(
args.environment, from_date.isoformat(), to_date.isoformat()))
request_logs = sumo_logic_client.search(
sumo_logic_query(args.environment), start_dt=from_date, end_dt=to_date, fields=REQUEST_FIELDS)
tmp_file = NamedTemporaryFile(mode='w', encoding='utf-8', delete=False)
logging.debug("Writing out logs to temporary file {}".format(tmp_file.name))
try:
for request in request_logs:
tmp_file.write(json.dumps(request, default=json_serialize) + '\n')
s3_client.upload_from_file(s3_bucket, s3_key, tmp_file.name)
finally:
logging.debug("Removing temporary file {}".format(tmp_file.name))
os.remove(tmp_file.name)
def test_main(self):
xml = """<record>
<datafield tag="999" ind1="C" ind2="5">
<subfield code="s">Test Journal Name,100,10</subfield>
</datafield>
</record>"""
xml_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR)
xml_temp_file.write(xml)
xml_temp_file.flush()
kb = "TEST JOURNAL NAME---Converted"
kb_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR)
kb_temp_file.write(kb)
kb_temp_file.flush()
dest_temp_fd, dest_temp_path = mkstemp(dir=CFG_TMPDIR)
try:
os.close(dest_temp_fd)
process = subprocess.Popen([self.bin_path, xml_temp_file.name,
'--kb', kb_temp_file.name,
'-o', dest_temp_path],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
process.wait()
transformed_xml = open(dest_temp_path).read()
self.assertXmlEqual(transformed_xml, """<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record><datafield ind1="C" ind2="5" tag="999"><subfield code="s">Converted,100,10</subfield></datafield></record>
</collection>""")
finally:
os.unlink(dest_temp_path)
def run(self):
active_view = self.window.active_view()
text = "\n\n".join(getSelectedText(active_view)).strip()
tf = NamedTemporaryFile(mode="w", delete=False)
try:
tf.write(text)
tf.close()
res = subprocess.check_output(["m4", tf.name],
stderr=subprocess.STDOUT,
cwd=os.path.dirname(os.path.abspath(active_view.file_name())))
res = res.decode('utf-8').replace('\r', '').strip()
panel_name = "m4expand.results"
panel = self.window.create_output_panel(panel_name)
self.window.run_command("show_panel", {"panel": "output." + panel_name})
panel.set_read_only(False)
panel.set_syntax_file(active_view.settings().get("syntax"))
panel.run_command("append", {"characters": res})
panel.set_read_only(True)
except Exception as e:
print("M4Expand - An error occurred: ", e)
finally:
os.unlink(tf.name)
def testLogfile(self):
"""Test logging into a logfile"""
f = NamedTemporaryFile(delete=False)
filename = f.name
try:
set_log_level("error") # avoid using the console logger
f.write(":-P\n")
f.close()
start_logfile(f.name, "devinfo")
log = getLogger("prosoda.test.integration.test_logger")
log.debug("Should not be in logfile! :-( ")
log.info("Should be in logfile :-) ")
log.devinfo("Should be in logfile :-) ")
log.warning("Should really be in logfile :-D ")
stop_logfile(f.name)
contents = file(f.name).read()
self.assertNotIn(":-(", contents)
self.assertNotIn(":-P", contents)
self.assertIn(":-)", contents)
self.assertIn(":-D", contents)
# Make sure no colour codes are leaked into the logfile
self.assertNotIn("\033", contents)
finally:
set_log_level("debug")
unlink(filename)
def compile_inline(self,data,ext):
"""
Compile inline css. Have to compile to a file, because some css compilers
may not output to stdout, but we know they all output to a file. It's a
little hackish, but you shouldn't be compiling in production anyway,
right?
"""
compiler = settings.COMPILER_FORMATS[ext]
try:
bin = compiler['binary_path']
except:
raise Exception("Path to CSS compiler must be included in COMPILER_FORMATS")
tmp_file = NamedTemporaryFile(mode='w',suffix=ext)
tmp_file.write(dedent(data))
tmp_file.flush()
path, ext = os.path.splitext(tmp_file.name)
tmp_css = ''.join((path,'.css'))
self.compile(path,compiler)
data = open(tmp_css,'r').read()
# cleanup
tmp_file.close()
os.remove(tmp_css)
return data
def tmpfile(stream, mode=None):
"""Context manager that writes a :class:`Stream` object to a named
temporary file and yield it's filename. Cleanup deletes from the temporary
file from disk.
Args:
stream (Stream): Stream object to write to disk as temporary file.
mode (int, optional): File mode to set on temporary file.
Returns:
str: Temporoary file name
"""
tmp = NamedTemporaryFile(delete=False)
if mode is not None:
oldmask = os.umask(0)
try:
os.chmod(tmp.name, mode)
finally:
os.umask(oldmask)
for data in stream:
tmp.write(to_bytes(data))
tmp.close()
yield tmp.name
os.remove(tmp.name)
def test_safe_md5(self):
"""Make sure we have the expected md5 with varied input types
This method is ported from PyCogent (http://www.pycogent.org). PyCogent
is a GPL project, but we obtained permission from the authors of this
method to port it to the BIOM Format project (and keep it under BIOM's
BSD license).
"""
exp = 'd3b07384d113edec49eaa6238ad5ff00'
tmp_f = NamedTemporaryFile(
mode='w',
prefix='test_safe_md5',
suffix='txt')
tmp_f.write('foo\n')
tmp_f.flush()
obs = safe_md5(open(tmp_f.name, 'U'))
self.assertEqual(obs, exp)
obs = safe_md5(['foo\n'])
self.assertEqual(obs, exp)
# unsupported type raises TypeError
self.assertRaises(TypeError, safe_md5, 42)
def run_solver(self, conflicts, election, deletion_handler, outfile=None):
if not conflicts:
return [], 0
self.deletion_handler = deletion_handler
instance = self.generate_instance(conflicts, election)
f = NamedTemporaryFile(delete=False)
f.write(instance.encode(code))
f.close()
process = Popen([self.cmd, f.name], stdout=PIPE)
out, err = process.communicate()
conflict_variables, optimum = self.parse_instance(out)
if outfile:
candidates = election[0]
votes = election[1]
votecounts = election[2]
votemap = self.delete_votes(votes, votecounts, conflict_variables)
votesum = sum(votemap.values())
write_map(candidates, votesum, votemap, open(outfile, "w"))
remove(f.name)
return conflict_variables, optimum
def odt_subreport(name=None, obj=None):
if not aeroo_ooo:
return _("Error! Subreports not available!")
report_xml_ids = ir_obj.search(cr, uid, [('report_name', '=', name)], context=context)
if report_xml_ids:
service = netsvc.Service._services['report.%s' % name]
report_xml = ir_obj.browse(cr, uid, report_xml_ids[0], context=context)
data = {'model': obj._table_name, 'id': obj.id, 'report_type': 'aeroo', 'in_format': 'oo-odt'}
### Get new printing object ###
sub_aeroo_print = AerooPrint()
service.active_prints[sub_aeroo_print.id] = sub_aeroo_print
context['print_id'] = sub_aeroo_print.id
###############################
sub_aeroo_print.start_time = time.time()
report, output = service.create_aeroo_report(cr, uid, \
[obj.id], data, report_xml, context=context,
output='odt') # change for OpenERP 6.0 - Service class usage
### Delete printing object ###
AerooPrint.print_ids.remove(sub_aeroo_print.id)
del service.active_prints[sub_aeroo_print.id]
##############################
temp_file = NamedTemporaryFile(suffix='.odt', prefix='aeroo-report-', delete=False)
try:
temp_file.write(report)
finally:
temp_file.close()
# self.oo_subreports[print_id].append(temp_file.name)
# aeroo_print.subreports.append(temp_file.name)
self.active_prints[aeroo_print.id].subreports.append(temp_file.name)
return "<insert_doc('%s')>" % temp_file.name
return None
def get_tempfile(self, **kwargs):
kwargs.setdefault('suffix', '.vrt')
tempfile = NamedTemporaryFile(**kwargs)
tempfile.write(self.content)
tempfile.flush()
tempfile.seek(0)
return tempfile
def render_to_temporary_file(self, template_name, mode='w+b', bufsize=-1,
suffix='.html', prefix='tmp', dir=None,
delete=True):
template = self.resolve_template(template_name)
context = self.resolve_context(self.context_data)
content = smart_str(template.render(context))
content = make_absolute_paths(content)
try:
tempfile = NamedTemporaryFile(mode=mode, bufsize=bufsize,
suffix=suffix, prefix=prefix,
dir=dir, delete=delete)
except TypeError:
tempfile = NamedTemporaryFile(mode=mode, buffering=bufsize,
suffix=suffix, prefix=prefix,
dir=dir, delete=delete)
try:
tempfile.write(content)
tempfile.flush()
return tempfile
except TypeError:
tempfile.write(bytes(content, 'UTF-8'))
tempfile.flush()
return tempfile
except:
# Clean-up tempfile if an Exception is raised.
tempfile.close()
raise
def create_temp_file(self, edid_binary):
edid_file = NamedTemporaryFile(delete=False)
edid_file.write(edid_binary)
edid_file.flush()
edid_file.seek(0)
return edid_file
def _write_local_schema_file(self, cursor):
"""
Takes a cursor, and writes the BigQuery schema for the results to a
local file system.
:return: A dictionary where key is a filename to be used as an object
name in GCS, and values are file handles to local files that
contains the BigQuery schema fields in .json format.
"""
schema = []
for field in cursor.description:
# See PEP 249 for details about the description tuple.
field_name = field[0]
field_type = self.type_map(field[1])
field_mode = 'REPEATED' if field[1] in (1009, 1005, 1007,
1016) else 'NULLABLE'
schema.append({
'name': field_name,
'type': field_type,
'mode': field_mode,
})
self.log.info('Using schema for %s: %s', self.schema_filename, schema)
tmp_schema_file_handle = NamedTemporaryFile(delete=True)
s = json.dumps(schema, sort_keys=True)
if PY3:
s = s.encode('utf-8')
tmp_schema_file_handle.write(s)
return {self.schema_filename: tmp_schema_file_handle}
def test_pipeline_run():
'It tests that the pipeline runs ok'
pipeline = 'sanger_with_qual'
fhand_adaptors = NamedTemporaryFile()
fhand_adaptors.write(ADAPTORS)
fhand_adaptors.flush()
arabidopsis_genes = 'arabidopsis_genes+'
univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
configuration = {'remove_vectors_blastdb': {'vectors': univec},
'remove_adaptors': {'adaptors': fhand_adaptors.name}}
seq_fhand = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r')
qual_fhand = open(os.path.join(TEST_DATA_DIR, 'qual.fasta'), 'r')
seq_iter = seqs_in_file(seq_fhand, qual_fhand)
filtered_seq_iter = _pipeline_builder(pipeline, seq_iter,
configuration)
seq_list = list(filtered_seq_iter)
assert 'CGAtcgggggg' in str(seq_list[0].seq)
assert len(seq_list) == 6
def _generate_training_files(self):
"""Returns a tuple of file objects suitable for passing to the
RdpTrainer application controller.
"""
tmp_dir = get_qiime_temp_dir()
training_set = RdpTrainingSet()
reference_seqs_file = open(self.Params['reference_sequences_fp'], 'U')
id_to_taxonomy_file = open(self.Params['id_to_taxonomy_fp'], 'U')
for seq_id, seq in MinimalFastaParser(reference_seqs_file):
training_set.add_sequence(seq_id, seq)
for line in id_to_taxonomy_file:
seq_id, lineage_str = map(strip, line.split('\t'))
training_set.add_lineage(seq_id, lineage_str)
training_set.dereplicate_taxa()
rdp_taxonomy_file = NamedTemporaryFile(
prefix='RdpTaxonAssigner_taxonomy_', suffix='.txt', dir=tmp_dir)
rdp_taxonomy_file.write(training_set.get_rdp_taxonomy())
rdp_taxonomy_file.seek(0)
rdp_training_seqs_file = NamedTemporaryFile(
prefix='RdpTaxonAssigner_training_seqs_', suffix='.fasta',
dir=tmp_dir)
for rdp_id, seq in training_set.get_training_seqs():
rdp_training_seqs_file.write('>%s\n%s\n' % (rdp_id, seq))
rdp_training_seqs_file.seek(0)
self._training_set = training_set
return rdp_taxonomy_file, rdp_training_seqs_file
def test_seq_pipeline_parallel_run(self):
'It tests that the pipeline runs ok'
pipeline = 'sanger_without_qual'
fhand_adaptors = NamedTemporaryFile()
fhand_adaptors.write(ADAPTORS)
fhand_adaptors.flush()
arabidopsis_genes = 'arabidopsis_genes+'
univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
configuration = {'remove_vectors': {'vectors': univec},
'remove_adaptors': {'adaptors': fhand_adaptors.name}}
in_fhands = {}
in_fhands['in_seq'] = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'),
'r')
out_fhand = NamedTemporaryFile()
writer = SequenceWriter(out_fhand, file_format='fasta')
writers = {'seq': writer}
seq_pipeline_runner(pipeline, configuration, in_fhands,
processes=4, writers=writers)
out_fhand = open(out_fhand.name, 'r')
result_seq = out_fhand.read()
assert result_seq.count('>') == 6
#are we keeping the description?
assert 'mdust' in result_seq
def test_seq_pipeline_parallel_run_with_fasta_qual(self):
'The pipeline runs in parallel with fasta and qual'
pipeline = 'sanger_with_qual'
fhand_adaptors = NamedTemporaryFile()
fhand_adaptors.write(ADAPTORS)
fhand_adaptors.flush()
arabidopsis_genes = 'arabidopsis_genes+'
univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
configuration = {'remove_vectors': {'vectors': univec},
'remove_adaptors': {'adaptors': fhand_adaptors.name}}
seq1 = create_random_seqwithquality(500, qual_range=50)
seq2 = create_random_seqwithquality(500, qual_range=51)
seq3 = create_random_seqwithquality(500, qual_range=52)
seqs = [seq1, seq2, seq3]
inseq_fhand, inqual_fhand = create_temp_seq_file(seqs, format='qual')
in_fhands = {}
in_fhands['in_seq'] = open(inseq_fhand.name)
in_fhands['in_qual'] = open(inqual_fhand.name)
outseq_fhand = NamedTemporaryFile()
outqual_fhand = NamedTemporaryFile()
writer = SequenceWriter(outseq_fhand, qual_fhand=outqual_fhand,
file_format='fasta')
writers = {'seq': writer}
seq_pipeline_runner(pipeline, configuration, in_fhands,
processes=4, writers=writers)
out_fhand = open(outseq_fhand.name, 'r')
result_seq = out_fhand.read()
assert result_seq.count('>') == 3
def write_temp_file(data):
# create a temp file for use as a config file. This should get cleaned
# up magically at the end of the run.
fid = NamedTemporaryFile(mode='w+b', suffix='.tmp')
fid.write(data)
fid.seek(0)
return fid
def test_toy_corpus():
keats = ('She dwells with Beauty - Beauty that must die;\n\n'
'And Joy, whose hand is ever at his lips\n\n'
'Bidding adieu; and aching Pleasure nigh,\n\n'
'Turning to poison while the bee-mouth sips:\n\n'
'Ay, in the very temple of Delight\n\n'
'Veil\'d Melancholy has her sovran shrine,\n\n'
'Though seen of none save him whose strenuous tongue\n\n'
'Can burst Joy\'s grape against his palate fine;\n\n'
'His soul shall taste the sadness of her might,\n\n'
'And be among her cloudy trophies hung.')
assert toy_corpus(keats)
assert toy_corpus(keats, nltk_stop=True)
assert toy_corpus(keats, stop_freq=1)
assert toy_corpus(keats, add_stop=['and', 'with'])
assert toy_corpus(keats, nltk_stop=True,
stop_freq=1, add_stop=['ay'])
import os
from tempfile import NamedTemporaryFile as NFT
tmp = NFT(delete=False)
tmp.write(keats)
tmp.close()
c = toy_corpus(tmp.name, is_filename=True,
nltk_stop=True, add_stop=['ay'])
assert c
os.remove(tmp.name)
return c
def _write_local_schema_file(self, cursor):
"""
Takes a cursor, and writes the BigQuery schema for the results to a
local file system.
:return: A dictionary where key is a filename to be used as an object
name in GCS, and values are file handles to local files that
contains the BigQuery schema fields in .json format.
"""
schema = []
for field in cursor.description:
# See PEP 249 for details about the description tuple.
field_name = field[0]
field_type = self.type_map(field[1])
# Always allow TIMESTAMP to be nullable. MySQLdb returns None types
# for required fields because some MySQL timestamps can't be
# represented by Python's datetime (e.g. 0000-00-00 00:00:00).
field_mode = 'NULLABLE' if field[6] or field_type == 'TIMESTAMP' else 'REQUIRED'
schema.append({
'name': field_name,
'type': field_type,
'mode': field_mode,
})
self.log.info('Using schema for %s: %s', self.schema_filename, schema)
tmp_schema_file_handle = NamedTemporaryFile(delete=True)
s = json.dumps(schema, tmp_schema_file_handle)
if PY3:
s = s.encode('utf-8')
tmp_schema_file_handle.write(s)
return {self.schema_filename: tmp_schema_file_handle}
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
# Collect logs from server
kwargs = {}
if self.namespace is not None:
kwargs['namespace'] = self.namespace
logs = self.osbs.get_build_logs(self.build_id, **kwargs)
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='w')
logfile.write(logs)
logfile.flush()
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='w')
docker_logs.write("\n".join(self.workflow.build_logs))
docker_logs.flush()
return [Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")),
Output(file=logfile,
metadata=self.get_output_metadata(logfile.name,
"openshift-final.log"))]
def testFileAnnotationSpeed(author_testimg_generated, gatewaywrapper):
""" Tests speed of loading file annotations. See PR: 4176 """
try:
f = NamedTemporaryFile()
f.write("testFileAnnotationSpeed text")
ns = TESTANN_NS
image = author_testimg_generated
# use the same file to create many file annotations
for i in range(20):
fileAnn = gatewaywrapper.gateway.createFileAnnfromLocalFile(f.name, mimetype="text/plain", ns=ns)
image.linkAnnotation(fileAnn)
finally:
f.close()
now = time.time()
for ann in image.listAnnotations():
if ann._obj.__class__ == omero.model.FileAnnotationI:
# mimmic behaviour of templates which call multiple times
print ann.getId()
print ann.getFileName()
print ann.getFileName()
print ann.getFileSize()
print ann.getFileSize()
print time.time() - now
def SubmitPlay(request):
import jinja2
from tempfile import NamedTemporaryFile
import os
html = ''
inventory = """
[current]
{{ public_ip_address }}
"""
# for (name,value) in request.GET:
# if name=='Servers':
# html=html+str(value+'\n')
html=request.GET['Severs']
inventory_template = jinja2.Template(inventory)
rendered_inventory = inventory_template.render({
'public_ip_address': html
# and the rest of our variables
})
# Create a temporary file and write the template string to it
hosts = NamedTemporaryFile(delete=False)
hosts.write(rendered_inventory)
hosts.close()
print(hosts.name)
import ansiblepythonapi as myPlay
args=['/home/ec2-user/playss/AnsiblePlus/test.yml']
# args.append('-i')
# args.append(hosts.name)
message=myPlay.main(args)
objects=[]
for runner_results in myPlay.message:
values=[]
for (host, value) in runner_results.get('dark', {}).iteritems():
try:
values.append(host)
values.append(value['failed'])
values.append(value['msg'])
objects.append(values)
except:
pass
for (host, value) in runner_results.get('contacted', {}).iteritems():
try:
values.append(host)
values.append(value['failed'])
values.append(value['msg'])
objects.append(values)
except:
pass
# for msg in pb.stats.output():
context=Context({'Summary':objects})
return render(request, 'AnsibleResponce.html',context)
def run_via_pbs(args, pbs):
assert(pbs in ('condor',)) # for now
# TODO: RF to support multiple backends, parameters, etc, for now -- just condor, no options
f = NamedTemporaryFile('w', prefix='datalad-%s-' % pbs, suffix='.submit', delete=False)
try:
pwd = getpwd()
logs = f.name.replace('.submit', '.log')
exe = args[0]
# TODO: we might need better way to join them, escaping spaces etc. There must be a stock helper
#exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else ''
exe_args = ' '.join(args[1:]) if len(args) > 1 else ''
f.write("""\
Executable = %(exe)s
Initialdir = %(pwd)s
Output = %(logs)s
Error = %(logs)s
getenv = True
arguments = %(exe_args)s
queue
""" % locals())
f.close()
Runner().run(['condor_submit', f.name])
lgr.info("Scheduled execution via %s. Logs will be stored under %s" % (pbs, logs))
finally:
os.unlink(f.name)
def hmmscan(fasta, database_path, ncpus=10):
F = NamedTemporaryFile()
F.write(fasta)
F.flush()
OUT = NamedTemporaryFile()
cmd = '%s --cpu %s -o /dev/null -Z 190000 --tblout %s %s %s' %(HMMSCAN, ncpus, OUT.name, database_path, F.name)
#print cmd
sts = subprocess.call(cmd, shell=True)
byquery = defaultdict(list)
if sts == 0:
for line in OUT:
#['#', '---', 'full', 'sequence', '----', '---', 'best', '1', 'domain', '----', '---', 'domain', 'number', 'estimation', '----']
#['#', 'target', 'name', 'accession', 'query', 'name', 'accession', 'E-value', 'score', 'bias', 'E-value', 'score', 'bias', 'exp', 'reg', 'clu', 'ov', 'env', 'dom', 'rep', 'inc', 'description', 'of', 'target']
#['#-------------------', '----------', '--------------------', '----------', '---------', '------', '-----', '---------', '------', '-----', '---', '---', '---', '---', '---', '---', '---', '---', '---------------------']
#['delNOG20504', '-', '553220', '-', '1.3e-116', '382.9', '6.2', '3.4e-116', '381.6', '6.2', '1.6', '1', '1', '0', '1', '1', '1', '1', '-']
if line.startswith('#'): continue
fields = line.split() # output is not tab delimited! Should I trust this split?
hit, _, query, _ , evalue, score, bias, devalue, dscore, dbias = fields[0:10]
evalue, score, bias, devalue, dscore, dbias = map(float, [evalue, score, bias, devalue, dscore, dbias])
byquery[query].append([hit, evalue, score])
OUT.close()
F.close()
return byquery
def _write_local_data_files(self, cursor):
"""
Takes a cursor, and writes results to a local file.
:return: A dictionary where keys are filenames to be used as object
names in GCS, and values are file handles to local files that
contain the data for the GCS objects.
"""
schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
file_no = 0
tmp_file_handle = NamedTemporaryFile(delete=True)
tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}
for row in cursor:
# Convert datetime objects to utc seconds, and decimals to floats
row = map(self.convert_types, row)
row_dict = dict(zip(schema, row))
# TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
s = json.dumps(row_dict)
if PY3:
s = s.encode('utf-8')
tmp_file_handle.write(s)
# Append newline to make dumps BigQuery compatible.
tmp_file_handle.write(b'\n')
# Stop if the file exceeds the file size limit.
if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
file_no += 1
tmp_file_handle = NamedTemporaryFile(delete=True)
tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle
return tmp_file_handles
def get_splice_score(a, s_type=5):
if s_type not in [3,5]:
raise Exception("Invalid splice type {}, should be 3 or 5".format(s_type))
maxent = config.maxentpath
if not maxent:
raise Exception("Please provide path to the score5.pl and score3.pl maxent scripts in config file")
tmp = NamedTemporaryFile()
for name,seq in a:
tmp.write(">{}\n{}\n".format(name,seq))
tmp.flush()
cmd = "perl score{}.pl {}".format(s_type, tmp.name)
p = sp.Popen(cmd, shell=True, cwd=maxent, stdout=sp.PIPE)
score = 0
for line in p.stdout.readlines():
vals = line.strip().split("\t")
if len(vals) > 1:
try:
score += float(vals[-1])
except ValueError:
logger.error("valueError, skipping: {}".format(vals))
except:
logger.error("Something unexpected happened")
return score
def _build_and_catch_errors(self, build_func, options_bytes, source=None):
try:
return build_func()
except _cl.RuntimeError as e:
msg = e.what
if options_bytes:
msg = msg + "\n(options: %s)" % options_bytes.decode("utf-8")
if source is not None:
from tempfile import NamedTemporaryFile
srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl")
try:
srcfile.write(source)
finally:
srcfile.close()
msg = msg + "\n(source saved as %s)" % srcfile.name
code = e.code
routine = e.routine
err = _cl.RuntimeError(
_cl.Error._ErrorRecord(
msg=msg,
code=code,
routine=routine))
# Python 3.2 outputs the whole list of currently active exceptions
# This serves to remove one (redundant) level from that nesting.
raise err
class GitIndexTests(unittest.TestCase):
def setUp(self):
self.dir = TemporaryDirectory()
self.file = NamedTemporaryFile(dir=self.dir.name, delete=False)
self.filename = path.basename(self.file.name)
self.author = Signature('QuitStoreTest', '*****@*****.**')
self.comitter = Signature('QuitStoreTest', '*****@*****.**')
self.repo = quit.git.Repository(self.dir.name, create=True)
def tearDown(self):
self.file = None
self.filename = None
self.dir.cleanup()
self.dir = None
self.reop = None
def addfile(self):
"""Create a repository and add a file to the git index."""
# Write to file
self.file.write(b'First Line\n')
self.file.read()
# Add file to index
repo = Repository(self.dir.name)
index = repo.index
index.read()
index.add(self.filename)
index.write()
self.repo = quit.git.Repository(self.dir.name)
def createcommit(self):
"""Prepare a git repository with one existing commit.
Create a directory, initialize a git Repository, add
and commit a file.
Returns:
A list containing the directory and file
"""
self.addfile()
# Create commit
repo = Repository(self.dir.name)
index = repo.index
index.read()
tree = index.write_tree()
message = "First commit of temporary test repo"
repo.create_commit('HEAD', self.author, self.comitter, message, tree,
[])
self.repo = quit.git.Repository(self.dir.name)
def testIndexSetRevision(self):
self.createcommit()
reviosions = self.repo.revisions()
index = quit.git.Index(self.repo)
index.set_revision(reviosions[0].id)
with self.assertRaises(Exception) as context:
index.set_revision('not.existing.revision')
def testIndexAddFile(self):
index = quit.git.Index(self.repo)
self.assertEqual(len(index.stash), 0)
index.add(self.filename, b'First Line\n')
self.assertEqual(len(index.stash), 1)
def testIndexCommit(self):
index = quit.git.Index(self.repo)
self.assertFalse(index.dirty)
commit = index.commit("First commit from quit test", "QuitTest",
"*****@*****.**")
self.assertTrue(index.dirty)
with self.assertRaises(Exception) as context:
index.commit("Second commit from quit test", "QuitTest",
"*****@*****.**")
def flushPartition(self):
# Flush a Partition to secondary memory.
# Choose a random victim from main memory, which size is not empty.
victim_len = 0
while (victim_len == 0):
victim = randint(
0, (self.left_table.size + self.right_table.size) - 1)
if (victim < self.left_table.size):
victim_len = len(self.left_table.partitions[victim].records)
else:
victim_len = len(
self.right_table.partitions[victim %
self.right_table.size].records)
if (victim < self.left_table.size):
file_descriptor = self.fileDescriptor_left
partition_to_flush = self.left_table.partitions[victim]
self.memory_left = self.memory_left - len(
partition_to_flush.records)
self.left_table.partitions[victim] = Partition()
else:
victim = victim % self.right_table.size
file_descriptor = self.fileDescriptor_right
partition_to_flush = self.right_table.partitions[victim]
self.memory_right = self.memory_right - len(
partition_to_flush.records)
self.right_table.partitions[victim] = Partition()
# Update file descriptor
if (file_descriptor.has_key(victim)):
lenfile = file_descriptor[victim].size
file = open(file_descriptor[victim].file.name, 'a')
file_descriptor.update({
victim:
FileDescriptor(file,
len(partition_to_flush.records) + lenfile,
file_descriptor[victim].timestamps)
})
else:
file = NamedTemporaryFile(suffix=".ht", prefix="", delete=False)
file_descriptor.update({
victim:
FileDescriptor(file, len(partition_to_flush.records), set())
})
# Flush partition in file.
for record in partition_to_flush.records:
self.timestamp += 1
tuple = str(record.tuple)
ats = str(record.ats)
dts = str(self.timestamp)
#ats = "%.40r" % (record.ats)
#dts = "%.40r" % (time())
file.write(tuple + '|')
file.write(ats + '|')
file.write(dts + '\n')
file.close()
class _TestPymo(unittest.TestCase):
"""Test class for pymo module
Example usage:
python pymo.py (to execute all tests)
python pymo.py -c (for verbose console logging)
python pymo.py -f mylog.log (for logging to file mylog.log)
python pymo.py -c -f mylog.log (for both)
python pymo.py _Test_pymo.test_fetch_smiles # (to execute only the specified test)
coverage run pymo.py (to run test code coverage analysis)
coverage report pymo.py (to view the result of the test code coverage analysis)
"""
def setUp(self):
"""setup test data location, unittest config and logger"""
# location of test data
self.test_data_location = root_dir + '/contrib/script/py/mmp/testdata/'
# temp output file and dir
self.temp_inp_file = NamedTemporaryFile(encoding='utf-8',
mode='wt',
suffix='.smi',
delete=False)
self.temp_out_file = NamedTemporaryFile(encoding='utf-8',
mode='wt',
delete=False)
self.temp_out_dir = mkdtemp()
test_smiles = {
# basic test set - all the below id's and structures are CHEMBL
'3105327': 'Cc1ccc2c(ccn2c3nc(cs3)c4cc(ccc4F)C(F)(F)F)c1',
'1526778': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccc(C)c(C)c3',
'1494678': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccccc3',
'472166': 'OC(CCn1ccnc1)(c2ccccc2)c3ccccc3',
'69798': 'Cc1nccn1CCC(O)(c2ccccc2)c3ccccc3',
'367346': 'Cc1sc(N)nc1c2cccc(Cl)c2',
'366881': 'Cc1sc(N)nc1c2ccc(Cl)c(Cl)c2',
'1477460': 'COc1ccc(cc1)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C',
'1441050': 'COc1ccc(cc1OC)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C'
}
# write test data to temp file 01
for smi_id, smi in test_smiles.items():
string = smi + ' ' + smi_id + '\n'
self.temp_inp_file.write(string)
self.temp_inp_file.close()
def tearDown(self):
"""cleanup test data and settings"""
# Clean up the directory
# os.removedirs(self.temp_out_dir)
shutil.rmtree(self.temp_out_dir)
def test_fileconv(self):
log.debug("Testing fileconv")
exit_status = fileconv(os.path.join(self.test_data_location,
self.temp_inp_file.name),
params_dict={
'-v':
'',
'-c':
'10',
'-C':
'20',
'-S':
os.path.join(self.temp_out_dir,
'atomcountfilter'),
'-o':
'smi'
})
log.debug("fileconv return code was: %s" % exit_status)
self.assertEqual(exit_status, 0)
def test_make_these_molecules(self):
log.debug("Testing _make_these_molecules")
# test data from mmp_enum_mols_from_pairs.py
context = NamedTemporaryFile(encoding='utf-8',
mode='wt',
suffix='.smi',
delete=False)
examp_context = "[1CH3]CCCCCC partOne_1\n[1CH3]CCCCCCC partOne_2\n[1CH3]CCCCCCCC partOne_3\n[1CH3]CCCCCCCCC partOne_4\n[1CH3]CCCCCCCCCC partOne_5"
context.write(examp_context)
context.close()
frags = NamedTemporaryFile(encoding='utf-8',
mode='wt',
suffix='.smi',
delete=False)
examp_frags = "[1OH]CCCCCC partTwo_1\n[1OH]CCCCCCC partTwo_2\n[1OH]CCCCCCCC partTwo_3\n[1OH]CCCCCCCCC partTwo_4\n[1OH]CCCCCCCCCC partTwo_5\n[1OH]CCCCCCCCCCC partTwo_6\n"
frags.write(examp_frags)
frags.close()
make_instr = NamedTemporaryFile(encoding='utf-8',
mode='wt',
delete=False)
examp_make_instr = "partOne_2 partTwo_3\npartOne_4 partTwo_5\n"
make_instr.write(examp_make_instr)
make_instr.close()
rxn = NamedTemporaryFile(encoding='utf-8',
mode='wt',
suffix='.rxn',
delete=False)
# this is the reaction specification that trxn needs to combine isotopically labelled mmp fragmentation points
single_rxn = "(0 Reaction\n (0 Scaffold\n (A C smarts \"[!0*]\")\n (A I isotope (0 0))\n )\n"
single_rxn += " (1 Sidechain\n (A C smarts \"[!0*]\")\n (A I isotope (0 0))\n (A I join (0 0))\n )\n)"
rxn.write(single_rxn)
rxn.close()
exit_status = make_these_molecules([context.name, frags.name],
make_instr.name, [rxn.name])
log.debug("make_these_molecules return code was: %s" % exit_status)
self.assertEqual(exit_status, 0)
def test_dicer(self):
log.debug("Testing dicer")
log.debug("Testing %s" % dicer.__name__)
exit_status = dicer(
os.path.join(self.test_data_location, self.temp_inp_file.name),
os.path.join(self.temp_out_dir, 'dicer.out'),
params_dict={},
)
log.debug("dicer return code was: %s" % exit_status)
self.assertEqual(exit_status, 0)
def test_alogp(self):
log.debug("Testing CMI")
exit_status = alogp(
os.path.join(self.test_data_location, self.temp_inp_file.name),
os.path.join(self.temp_out_dir, 'alogp.out'))
log.debug("CMI return code was: %s" % exit_status)
self.assertEqual(exit_status, 0)
def mrjob(options):
"Generates and executes MR job script"
user = os.getenv('USER')
tstamp = int(time.time())
hdir = hdfs_dir(options.hdir, options.hdfs_prefix)
if PYDOOP:
odir = hdfs.path.join(hdir, options.odir)
idir = hdfs.path.join(hdir, options.idir)
schema = hdfs.path.join(hdir, options.schema)
for name in [
hdir,
odir,
idir,
]:
if options.verbose:
print("Checking %s" % name)
if not hdfs.path.isdir(name):
if name in [hdir, idir]:
print("ERROR: %s does not exist" % name)
sys.exit(1)
# else:
# print(" Creating output directory: %s" % name)
# hdfs.mkdir(name)
elif name == odir:
# in case odir exists and is not empty, move it somewhere and re-create
if hdfs.ls(odir):
ocache = hdfs.path.normpath(odir) + '_%d' % tstamp
if options.verbose:
print(
" Non-empty output directory exists, saving it in %s"
% ocache)
hdfs.move(odir, ocache)
# hdfs.mkdir(odir)
# if it's empty, remove it
else:
hdfs.rmr(odir)
if options.verbose:
print("Checking %s" % schema)
if not hdfs.path.isfile(schema):
print("ERROR: %s does not exist" % schema)
sys.exit(1)
else:
idir = '%s%s' % (hdir, 'data')
odir = '%s%s' % (hdir, 'mrout')
schema = '%s%s' % (hdir, options.schema)
if options.verbose:
msg = 'pydoop module is not present on this system'
msg += ', will use input as is without checking'
print('WARNING:', msg)
for name in [options.mrpy, options.pydoop, options.avro]:
if options.verbose:
print("Checking %s" % name)
if not os.path.isfile(name):
print("ERROR: %s does not exist" % name)
sys.exit(1)
# module = os.path.basename(os.path.splitext(options.mrpy)[0])
code = create_mrpy(options.mrpy, options.verbose)
cmd = """#!/bin/bash
input={input}
output={output}
schema={schema}
ifile=/tmp/mr_{user}_{tstamp}.py
cat << EOF > $ifile
{code}
EOF
module=mr_{user}_{tstamp}
arch_pydoop={pydoop}
arch_avro={avro}
echo "Input URI : $input"
echo "Output URI: $output"
echo "Schema: $schema"
echo "MR script : $ifile"
echo "Module name : $module"
echo "Pydoop archive: $arch_pydoop"
echo "Avro archive : $arch_avro"
echo "-----------------"
echo "Submitting MR job"
pydoop submit \
--upload-archive-to-cache $arch_pydoop \
--upload-archive-to-cache $arch_avro \
-D avro.schema=$schema \
--do-not-use-java-record-reader \
--log-level {loglevel} \
--job-name WMArchive \
--num-reducers 1 \
--upload-file-to-cache $ifile \
--mrv2 $module $input $output
""".format(input=idir,
output=odir,
user=user,
tstamp=tstamp,
code=code,
schema=schema,
loglevel=options.loglevel,
pydoop=os.path.abspath(options.pydoop),
avro=os.path.abspath(options.avro))
fobj = NamedTemporaryFile(delete=False)
fobj.write(cmd)
fobj.close()
fstat = os.stat(fobj.name)
os.chmod(fobj.name, fstat.st_mode | stat.S_IEXEC)
if options.execute:
run(fobj.name, options.verbose)
else:
if options.verbose:
print("------- Generated script --------")
print(open(fobj.name, 'r').read())
if options.verbose:
print("---------------------------------")
# clean up temporary file
os.unlink(fobj.name)
def run_p2p_prediction(tx_lat, tx_lng, rx_lat, rx_lng, path_ssn,
path_name="",
tx_antenna="ISOTROPIC",
tx_gos=0.0,
rx_antenna="ISOTROPIC",
rx_gos=0.0,
path_month=None,
path_year=None,
path_hour=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24],
path_frequency=[10],
path_bw=3000,
path_SNRr=15,
tx_power=100,
path_sorl="SHORTPATH",
path_manmade_noise="CITY",
report_format=["RPT_BCR"],
data_path="./data/",
report_dict_keys=['BCR'],
zeroMidnight=False,
returnInputFile=False,
returnOutputFile=False
):
tx_power = 10 * (log10(tx_power/1000.0))
report_format_str = " | ".join(report_format)
path_frequency_str = ", ".join([str(n) for n in path_frequency])
path_hour_str = ", ".join([str(n) for n in path_hour])
buf = []
if path_name:
buf.append('PathName "{:s}"'.format(path_name))
buf.append('Path.L_tx.lat {:.6f}'.format(tx_lat))
buf.append('Path.L_tx.lng {:.6f}'.format(tx_lng))
buf.append('TXAntFilePath "{:s}"'.format(tx_antenna))
if tx_antenna == "ISOTROPIC":
buf.append('TXGOS {:.2f}'.format(tx_gos))
#buf.append('AntennaOrientation TX2RX')
buf.append('Path.L_rx.lat {:.6f}'.format(rx_lat))
buf.append('Path.L_rx.lng {:.6f}'.format(rx_lng))
buf.append('RXAntFilePath "{:s}"'.format(rx_antenna))
if rx_antenna == "ISOTROPIC":
buf.append('RXGOS {:.2f}'.format(rx_gos))
#buf.append('AntennaOrientation RX2TX')
if not path_year:
now = datetime.datetime.utcnow()
path_year = now.year
buf.append('Path.year {:d}'.format(path_year))
if not path_month:
now = datetime.datetime.utcnow()
path_month = now.month
buf.append('Path.month {:d}'.format(path_month))
buf.append('Path.hour {:s}'.format(path_hour_str))
buf.append('Path.SSN {:d}'.format(path_ssn))
buf.append('Path.frequency {:s}'.format(path_frequency_str))
buf.append('Path.txpower {:.2f}'.format(tx_power))
buf.append('Path.BW {:.2f}'.format(path_bw))
buf.append('Path.SNRr {:.2f}'.format(path_SNRr))
buf.append('Path.SNRXXp 90')
buf.append('Path.ManMadeNoise "{:s}"'.format(path_manmade_noise))
buf.append('Path.SorL "{:s}"'.format(path_sorl))
buf.append('RptFileFormat "{:s}"'.format(report_format_str))
buf.append('LL.lat {:.6f}'.format(rx_lat))
buf.append('LL.lng {:.6f}'.format(rx_lng))
buf.append('LR.lat {:.6f}'.format(rx_lat))
buf.append('LR.lng {:.6f}'.format(rx_lng))
buf.append('UL.lat {:.6f}'.format(rx_lat))
buf.append('UL.lng {:.6f}'.format(rx_lng))
buf.append('UR.lat {:.6f}'.format(rx_lat))
buf.append('UR.lng {:.6f}'.format(rx_lng))
buf.append('DataFilePath "{:s}"'.format(data_path))
input_file = NamedTemporaryFile(mode='w+t', prefix="proppy_", suffix='.in', delete=False)
text_in = "{:s}\n".format('\n'.join(buf))
#print(text_in)
input_file.write(text_in)
input_file.close()
FNULL = open(os.devnull, 'w')
output_file = NamedTemporaryFile(prefix="proppy_", suffix='.out', delete=False)
return_code = subprocess.call(["ITURHFProp",
'-c',
input_file.name,
output_file.name],
stderr=subprocess.STDOUT)
if return_code != 232:
raise ITURHFPropError("Internal Server Error: Return Code {:d}".format(return_code))
try:
prediction_dict = get_predictions_as_dict(output_file.name, report_dict_keys, zeroMidnight=zeroMidnight)
except:
print("Unexpected error:", sys.exc_info()[0])
raise ITURHFPropError("Internal Server Error: Error parsing file")
print(input_file.name)
print(output_file.name)
#os.remove(input_file.name)
#os.remove(output_file.name)
return prediction_dict
def get_readable_fileobj(name_or_obj,
encoding=None,
cache=False,
show_progress=True,
remote_timeout=None):
"""
Given a filename or a readable file-like object, return a context
manager that yields a readable file-like object.
This supports passing filenames, URLs, and readable file-like
objects, any of which can be compressed in gzip or bzip2.
Notes
-----
This function is a context manager, and should be used for example
as::
with get_readable_fileobj('file.dat') as f:
contents = f.read()
Parameters
----------
name_or_obj : str or file-like object
The filename of the file to access (if given as a string), or
the file-like object to access.
If a file-like object, it must be opened in binary mode.
encoding : str, optional
When `None` (default), returns a file-like object with a
`read` method that on Python 2.x returns `bytes` objects and
on Python 3.x returns `str` (`unicode`) objects, using
`locale.getpreferredencoding()` as an encoding. This matches
the default behavior of the built-in `open` when no `mode`
argument is provided.
When `'binary'`, returns a file-like object where its `read`
method returns `bytes` objects.
When another string, it is the name of an encoding, and the
file-like object's `read` method will return `str` (`unicode`)
objects, decoded from binary using the given encoding.
cache : bool, optional
Whether to cache the contents of remote URLs.
show_progress : bool, optional
Whether to display a progress bar if the file is downloaded
from a remote server. Default is `True`.
remote_timeout : float
Timeout for remote requests in seconds (default is the configurable
REMOTE_TIMEOUT, which is 3s by default)
Returns
-------
file : readable file-like object
"""
# close_fds is a list of file handles created by this function
# that need to be closed. We don't want to always just close the
# returned file handle, because it may simply be the file handle
# passed in. In that case it is not the responsibility of this
# function to close it: doing so could result in a "double close"
# and an "invalid file descriptor" exception.
close_fds = []
delete_fds = []
if remote_timeout is None:
# use configfile default
remote_timeout = REMOTE_TIMEOUT()
# Get a file object to the content
if isinstance(name_or_obj, six.string_types):
if _is_url(name_or_obj):
name_or_obj = download_file(name_or_obj,
cache=cache,
show_progress=show_progress,
timeout=remote_timeout)
if six.PY3:
fileobj = io.FileIO(name_or_obj, 'r')
elif six.PY2:
fileobj = open(name_or_obj, 'rb')
close_fds.append(fileobj)
else:
fileobj = name_or_obj
# Check if the file object supports random access, and if not,
# then wrap it in a BytesIO buffer. It would be nicer to use a
# BufferedReader to avoid reading loading the whole file first,
# but that is not compatible with streams or urllib2.urlopen
# objects on Python 2.x.
if not hasattr(fileobj, 'seek'):
fileobj = io.BytesIO(fileobj.read())
# Now read enough bytes to look at signature
signature = fileobj.read(4)
fileobj.seek(0)
if signature[:3] == b'\x1f\x8b\x08': # gzip
import struct
try:
from .compat import gzip
fileobj_new = gzip.GzipFile(fileobj=fileobj, mode='rb')
fileobj_new.read(1) # need to check that the file is really gzip
except (IOError, EOFError): # invalid gzip file
fileobj.seek(0)
fileobj_new.close()
except struct.error: # invalid gzip file on Python 3
fileobj.seek(0)
fileobj_new.close()
else:
fileobj_new.seek(0)
fileobj = fileobj_new
elif signature[:3] == b'BZh': # bzip2
try:
# bz2.BZ2File does not support file objects, only filenames, so we
# need to write the data to a temporary file
tmp = NamedTemporaryFile("wb", delete=False)
tmp.write(fileobj.read())
tmp.close()
delete_fds.append(tmp)
import bz2
fileobj_new = bz2.BZ2File(tmp.name, mode='rb')
fileobj_new.read(1) # need to check that the file is really bzip2
except IOError: # invalid bzip2 file
fileobj.seek(0)
fileobj_new.close()
else:
fileobj_new.seek(0)
close_fds.append(fileobj_new)
fileobj = fileobj_new
# By this point, we have a file, io.FileIO, gzip.GzipFile, or
# bz2.BZ2File instance opened in binary mode (that is, read
# returns bytes). Now we need to, if requested, wrap it in a
# io.TextIOWrapper so read will return unicode based on the
# encoding parameter.
if six.PY3:
needs_textio_wrapper = encoding != 'binary'
elif six.PY2:
needs_textio_wrapper = encoding != 'binary' and encoding is not None
if needs_textio_wrapper:
# A bz2.BZ2File can not be wrapped by a TextIOWrapper,
# so we decompress it to a temporary file and then
# return a handle to that.
import bz2
if isinstance(fileobj, bz2.BZ2File):
tmp = NamedTemporaryFile("wb", delete=False)
data = fileobj.read()
tmp.write(data)
tmp.close()
delete_fds.append(tmp)
if six.PY3:
fileobj = io.FileIO(tmp.name, 'r')
elif six.PY2:
fileobj = open(tmp.name, 'rb')
close_fds.append(fileobj)
# On Python 2.x, we need to first wrap the regular `file`
# instance in a `io.FileIO` object before it can be
# wrapped in a `TextIOWrapper`. We don't just create an
# `io.FileIO` object in the first place, because we can't
# get a raw file descriptor out of it on Python 2.x, which
# is required for the XML iterparser.
if six.PY2 and isinstance(fileobj, file):
fileobj = io.FileIO(fileobj.fileno())
fileobj = io.BufferedReader(fileobj)
fileobj = io.TextIOWrapper(fileobj, encoding=encoding)
# Ensure that file is at the start - io.FileIO will for
# example not always be at the start:
# >>> import io
# >>> f = open('test.fits', 'rb')
# >>> f.read(4)
# 'SIMP'
# >>> f.seek(0)
# >>> fileobj = io.FileIO(f.fileno())
# >>> fileobj.tell()
# 4096L
fileobj.seek(0)
try:
yield fileobj
finally:
for fd in close_fds:
fd.close()
for fd in delete_fds:
os.remove(fd.name)
def ccopen(source, *args, **kwargs):
"""Guess the identity of a particular log file and return an instance of it.
Inputs:
source - a single logfile, a list of logfiles (for a single job),
an input stream, or an URL pointing to a log file.
*args, **kwargs - arguments and keyword arguments passed to filetype
Returns:
one of ADF, DALTON, GAMESS, GAMESS UK, Gaussian, Jaguar,
Molpro, MOPAC, NWChem, ORCA, Psi3, Psi/Psi4, QChem, CJSON or None
(if it cannot figure it out or the file does not exist).
"""
inputfile = None
is_stream = False
# Check if source is a link or contains links. Retrieve their content.
# Try to open the logfile(s), using openlogfile, if the source is a string (filename)
# or list of filenames. If it can be read, assume it is an open file object/stream.
is_string = isinstance(source, str)
is_url = True if is_string and URL_PATTERN.match(source) else False
is_listofstrings = isinstance(source, list) and all(
[isinstance(s, str) for s in source])
if is_string or is_listofstrings:
# Process links from list (download contents into temporary location)
if is_listofstrings:
filelist = []
for filename in source:
if not URL_PATTERN.match(filename):
filelist.append(filename)
else:
try:
response = urlopen(filename)
tfile = NamedTemporaryFile(delete=False)
tfile.write(response.read())
# Close the file because Windows won't let open it second time
tfile.close()
filelist.append(tfile.name)
# Delete temporary file when the program finishes
atexit.register(os.remove, tfile.name)
except (ValueError, URLError) as error:
if not kwargs.get('quiet', False):
(errno, strerror) = error.args
return None
source = filelist
if not is_url:
try:
inputfile = logfileparser.openlogfile(source)
except IOError as error:
if not kwargs.get('quiet', False):
(errno, strerror) = error.args
return None
else:
try:
response = urlopen(source)
is_stream = True
# Retrieve filename from URL if possible
filename = re.findall("\w+\.\w+", source.split('/')[-1])
filename = filename[0] if filename else ""
inputfile = logfileparser.openlogfile(filename,
object=response.read())
except (ValueError, URLError) as error:
if not kwargs.get('quiet', False):
(errno, strerror) = error.args
return None
elif hasattr(source, "read"):
inputfile = source
is_stream = True
# Streams are tricky since they don't have seek methods or seek won't work
# by design even if it is present. We solve this now by reading in the
# entire stream and using a StringIO buffer for parsing. This might be
# problematic for very large streams. Slow streams might also be an issue if
# the parsing is not instantaneous, but we'll deal with such edge cases
# as they arise. Ideally, in the future we'll create a class dedicated to
# dealing with these issues, supporting both files and streams.
if is_stream:
try:
inputfile.seek(0, 0)
except (AttributeError, IOError):
contents = inputfile.read()
try:
inputfile = io.StringIO(contents)
except:
inputfile = io.StringIO(unicode(contents))
inputfile.seek(0, 0)
# Proceed to return an instance of the logfile parser only if the filetype
# could be guessed. Need to make sure the input file is closed before creating
# an instance, because parsers will handle opening/closing on their own.
filetype = guess_filetype(inputfile)
# If the input file isn't a standard compchem log file, try one of
# the readers, falling back to Open Babel.
if not filetype:
if kwargs.get("cjson"):
filetype = readerclasses['cjson']
elif source and not is_stream:
ext = os.path.splitext(source)[1][1:].lower()
for extension in readerclasses:
if ext == extension:
filetype = readerclasses[extension]
# Proceed to return an instance of the logfile parser only if the filetype
# could be guessed. Need to make sure the input file is closed before creating
# an instance, because parsers will handle opening/closing on their own.
if filetype:
# We're going to close and reopen below anyway, so this is just to avoid
# the missing seek method for fileinput.FileInput. In the long run
# we need to refactor to support for various input types in a more
# centralized fashion.
if is_listofstrings:
pass
else:
inputfile.seek(0, 0)
if not is_stream:
if is_listofstrings:
if filetype == Turbomole:
source = sort_turbomole_outputs(source)
inputfile.close()
return filetype(source, *args, **kwargs)
return filetype(inputfile, *args, **kwargs)
def submit(self, **kwargs):
"""
dictionary of keywords to be parsed
"""
executable = kwargs.get("executable")
env = dict(kwargs.get("env", {}))
memory = kwargs.get("memory", 0.)
cpu = kwargs.get("cpu", 0.)
dry = bool(kwargs.get("dry", False))
verbose = bool(kwargs.get("verbose", True))
wd = str(kwargs.get("workdir", "$(pwd)"))
part = str(kwargs.get("partition", "debug"))
#log.error("ENV SETTINGS: %s",str(env))
if cpu == 0.: raise Exception("must provide cpu time")
sscript = NamedTemporaryFile(dir=wd,
suffix=".sh",
mode="w",
delete=False)
sname = abspath(sscript.name)
sscript.write("#!/bin/sh\n")
sscript.write("#SBATCH --ntasks=1\n")
sscript.write("#SBATCH --partition={part}\n".format(part=part))
sscript.write("#SBATCH -e {exe}.err\n".format(exe=sname))
sscript.write("#SBATCH -o {exe}.out\n".format(exe=sname))
sscript.write("#SBATCH --time={cpu}\n#SBATCH --mem={mem}\n\n".format(
cpu=cpu, mem=memory))
for key, value in env.iteritems():
environ[key] = value
sscript.write("sbatch --export={key} # {value}\n".format(
key=key, value=value))
sscript.write(
"\nsrun bash {executable}\n".format(executable=executable))
sscript.close()
chmod(sname, 0o755)
chdir(dirname(sname))
cmd = "{sub} ./{fn}".format(sub=self.executor, fn=basename(sname))
return self.__submit__(cmd, verbose=verbose, dry=dry)
class JoinPairedEndsTests(TestCase):
"""Tests for join_paired_ends."""
def setUp(self):
"""set up files to be used in all tests"""
# set up temp directory that all temp files will
# be written to:
self.temp_dir_path = mkdtemp()
# store files:
# joined_pairs
self.joined_pe = NamedTemporaryFile(prefix='joined_',
suffix='.fastq',
dir=self.temp_dir_path,
delete=False)
self.jpe_fp = self.joined_pe.name
self.joined_pe.write(joined_reads)
self.joined_pe.close()
# all barcodes
self.all_barcodes = NamedTemporaryFile(prefix='all_bc_',
suffix='.fastq',
dir=self.temp_dir_path,
delete=False)
self.all_bc_fp = self.all_barcodes.name
self.all_barcodes.write(all_barcodes)
self.all_barcodes.close()
# out of order barcodes
self.ooo_barcodes = NamedTemporaryFile(prefix='ooo_bc_',
suffix='.fastq',
dir=self.temp_dir_path,
delete=False)
self.ooo_bc_fp = self.ooo_barcodes.name
self.ooo_barcodes.write(all_barcodes_out_of_order)
self.ooo_barcodes.close()
# missing barcodes
self.missing_barcodes = NamedTemporaryFile(prefix='missing_bc_',
suffix='.fastq',
dir=self.temp_dir_path,
delete=False)
self.missing_bc_fp = self.missing_barcodes.name
self.missing_barcodes.write(missing_barcodes)
self.missing_barcodes.close()
def tearDown(self):
"""Remove all temp files"""
shutil.rmtree(self.temp_dir_path)
def test_write_synced_barcodes_fastq(self):
"""write_synced_barcodes_fastq: should work properly.
This function expects the barcodes.fastq and joined_pairs.fastq
files to be in the same order except for those missing data
that could not be joined. That is, not all paired-ends will
assemble.
"""
filtered_bc_path = write_synced_barcodes_fastq(self.jpe_fp,
self.all_bc_fp)
observed_barcodes = open(filtered_bc_path, 'U').read()
self.assertEqual(observed_barcodes, synced_barcodes)
os.remove(filtered_bc_path)
def test_out_of_order_barcodes(self):
"""write_synced_barcodes_fastq: should fail if barcodes out of order
By out of order I mean that the data in the barcodes file
are not in the same order as the data within the joined paired-ends
file.
"""
self.assertRaises(StopIteration, write_synced_barcodes_fastq,
self.jpe_fp, self.ooo_bc_fp)
def test_missing_barcodes(self):
""""write_synced_barcodes_fastq: should fail if barcodes are missing."""
self.assertRaises(StopIteration, write_synced_barcodes_fastq,
self.jpe_fp, self.missing_bc_fp)
class GitRepositoryTests(unittest.TestCase):
def setUp(self):
self.dir = TemporaryDirectory()
self.remotedir = TemporaryDirectory()
self.file = NamedTemporaryFile(dir=self.dir.name, delete=False)
self.filename = path.basename(self.file.name)
self.author = Signature('QuitStoreTest', '*****@*****.**')
self.comitter = Signature('QuitStoreTest', '*****@*****.**')
# Initialize repository
init_repository(self.dir.name, False)
def tearDown(self):
self.file = None
self.filename = None
self.dir.cleanup()
self.dir = None
self.remotedir.cleanup()
self.remotedir = None
def addfile(self):
"""Create a repository and add a file to the git index."""
# Write to file
self.file.write(b'First Line\n')
self.file.read()
# Add file to index
repo = Repository(self.dir.name)
index = repo.index
index.read()
index.add(self.filename)
index.write()
def createcommit(self):
"""Prepare a git repository with one existing commit.
Create a directory, initialize a git Repository, add
and commit a file.
Returns:
A list containing the directory and file
"""
self.addfile()
# Create commit
repo = Repository(self.dir.name)
index = repo.index
index.read()
tree = index.write_tree()
message = "First commit of temporary test repo"
repo.create_commit('HEAD', self.author, self.comitter, message, tree,
[])
def testInitNotExistingsRepo(self):
dir = TemporaryDirectory()
repo = quit.git.Repository(dir.name, create=True)
self.assertFalse(repo.is_bare)
self.assertEqual(len(repo.revisions()), 0)
dir.cleanup()
def testInitEmptyRepo(self):
self.addfile()
repo = quit.git.Repository(self.dir.name, create=True)
self.assertFalse(repo.is_bare)
self.assertEqual(len(repo.revisions()), 0)
def testInitRepoWithExistingCommit(self):
self.createcommit()
repo = quit.git.Repository(self.dir.name)
self.assertFalse(repo.is_bare)
self.assertEqual(len(repo.revisions()), 1)
def testCloneRepo(self):
REMOTE_NAME = 'origin'
REMOTE_URL = 'git://github.com/AKSW/QuitStore.example.git'
dir = TemporaryDirectory()
repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
self.assertTrue(path.exists(path.join(dir.name, 'example.nt')))
self.assertTrue(path.exists(path.join(dir.name, 'example.nt.graph')))
self.assertFalse(repo.is_bare)
dir.cleanup()
@unittest.skip("Currently fails on travis")
def testCloneRepoViaSSH(self):
environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/"
REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git'
dir = TemporaryDirectory()
repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
self.assertTrue(path.exists(path.join(dir.name, 'example.nt')))
self.assertFalse(repo.is_bare)
dir.cleanup()
def testCloneRepoViaSSHNoKeyFiles(self):
environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/nosshkey/"
if "SSH_AUTH_SOCK" in environ:
del environ["SSH_AUTH_SOCK"]
REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git'
dir = TemporaryDirectory()
with self.assertRaises(Exception) as context:
quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
dir.cleanup()
def testCloneNotExistingRepo(self):
environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/"
REMOTE_URL = '[email protected]:AKSW/ThereIsNoQuitStoreRepo.git'
dir = TemporaryDirectory()
with self.assertRaises(Exception) as context:
quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
dir.cleanup()
def testPushRepo(self):
"""Test if it is possible to push to an empty remote repository."""
with TemporaryRepository(True) as remote:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
quitRepo.push("origin", "master")
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
def testPushRefspecs(self):
"""Test if it is possible to push to an empty remote repository."""
for refspec in [
'master', 'refs/heads/master', 'refs/heads/master:master',
'master:master', 'master:refs/heads/master',
'refs/heads/master:refs/heads/master'
]:
with TemporaryRepository(True) as remote:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
quitRepo.push("origin", refspec)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
def testPushRepoNotConfiguredRemote(self):
"""Test if the push failes if the origin remote was not defined."""
with TemporaryRepository(True) as remote:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("upstream", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
with self.assertRaises(RemoteNotFound):
quitRepo.push("origin", "master")
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
def testPushRepoWithRemoteName(self):
"""Test if it is possible to push to a remote repository, which is not called orign."""
with TemporaryRepository(True) as remote:
graphContent = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("upstream", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
quitRepo.push("upstream", "master")
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
def testPushRepoNotConfiguredNamedRemote(self):
"""Test if the push failes if the specified remote was not defined."""
with TemporaryRepository(is_bare=True) as remote:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
with self.assertRaises(RemoteNotFound):
quitRepo.push("upstream", "master")
self.assertTrue(remote.is_empty)
self.assertFalse(local.is_empty)
def testPushRepoWithDivergedRemote(self):
"""Test for an exception, if the local and remote repositories are diverged."""
with TemporaryRepositoryFactory().withEmptyGraph(
"http://example.org/") as remote:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph(
"http://example.org/", graphContent) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
with self.assertRaises(pygit2.GitError):
quitRepo.push("origin", "master")
@unittest.skip("requires a remote with pre-receive hook")
def testPushRepoWithRemoteReject(self):
"""Test for an exception, if the remote repositories rejects a push.
CAUTION: This test is disabled, because it requires a remote with pre-receive hook.
Unfortunately the libgit2 does not execute pre-receive hooks on local repositories.
"""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as local:
local.remotes.create("origin", "ssh://[email protected]/testing.git")
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(local.is_empty)
with self.assertRaises(QuitGitPushError):
quitRepo.push()
def testFetchRepo(self):
"""Test if it is possible to fetch from a remote repository."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(False) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
with self.assertRaises(RevisionNotFound):
quitRepo.revision('HEAD')
quitRepo.fetch()
self.assertEqual(
quitRepo.revision('origin/master').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
def testFetchUpstreamRepo(self):
"""Test if it is possible to from from a remote, which set as upstream."""
graphContent = """
<http://ex.org/x> <http://ex.org/x> <http://ex.org/x> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(clone_from_repo=remote) as local:
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
with open(path.join(remote.workdir, "graph.nt"),
"a") as graphFile:
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
graphFile.write(graphContent)
createCommit(repository=remote)
remoteHead = remote.revparse_single('HEAD').hex
localHead = local.revparse_single('HEAD').hex
self.assertNotEqual(localHead, remoteHead)
quitRepo.fetch()
self.assertEqual(
quitRepo.revision('origin/master').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
def testPullRepo(self):
"""Test if it is possible to pull from a remote repository."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(False) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
with self.assertRaises(RevisionNotFound):
quitRepo.revision('HEAD')
quitRepo.pull("origin", "master")
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
def testPullRepoWithUnbornHead(self):
"""Test if it is possible to pull from a remote repository."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(False) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
with self.assertRaises(RevisionNotFound):
quitRepo.revision('HEAD')
quitRepo.pull()
self.assertEqual(
quitRepo.revision('origin/master').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
def testPullRepoClonedNoChanges(self):
"""Test pull if both repos are at the same state."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryDirectory() as localDirectory:
quitRepo = quit.git.Repository(localDirectory,
create=True,
origin=remote.path)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
quitRepo.pull("origin", "master")
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
def testPullRepoClonedAndPullWithChanges(self):
"""Test clone, commit on remote and pull."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryDirectory() as localDirectory:
quitRepo = quit.git.Repository(localDirectory,
create=True,
origin=remote.path)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
quitRepo.pull()
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteQuitRepo = quit.git.Repository(remote.workdir)
index = remoteQuitRepo.index(remoteHead)
graphContent += """
<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> ."""
index.add("graph.nt", graphContent)
author = Signature('QuitStoreTest', '*****@*****.**')
commitid = index.commit("from test", author.name, author.email)
quitRepo.pull()
self.assertEqual(quitRepo.revision('HEAD').id, str(commitid))
def testPullRepoClonedAndPullWithThreeWayGitMerge(self):
"""Test clone, commit on remote and pull with merge, which resolves without conflicts."""
graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryDirectory() as localDirectory:
quitRepo = quit.git.Repository(localDirectory,
create=True,
origin=remote.path)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
index = quitRepo.index(remoteHead)
graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
index.add("graph2.nt", graph2Content)
index.add("graph2.nt.graph", "http://example2.org/")
author = Signature('QuitStoreTest', '*****@*****.**')
localCommitid = index.commit("from local", author.name,
author.email)
quitRepo._repository.checkout_tree(
quitRepo._repository.get(localCommitid))
self.assertEqual(
quitRepo.revision('HEAD').id, str(localCommitid))
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteQuitRepo = quit.git.Repository(remote.workdir)
index = remoteQuitRepo.index(remoteHead)
graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
index.add("graph.nt", graphContent)
remoteCommitid = index.commit("from remote", author.name,
author.email)
remoteQuitRepo._repository.checkout_tree(
remoteQuitRepo._repository.get(remoteCommitid))
quitRepo.pull(method="three-way-git")
self.assertNotEqual(
quitRepo.revision('HEAD').id, str(localCommitid))
self.assertNotEqual(
quitRepo.revision('HEAD').id, str(remoteCommitid))
# check if head has local and remote commit id as ancestor
self.assertListEqual([
parent.id for parent in quitRepo.revision('HEAD').parents
], [str(localCommitid),
str(remoteCommitid)])
# check if the merged commit contains all file contents
with open(path.join(localDirectory, "graph.nt")) as f:
self.assertEqual("".join(f.readlines()), graphContent)
with open(path.join(localDirectory, "graph2.nt")) as f:
self.assertEqual("".join(f.readlines()), graph2Content)
def testPullRepoClonedAndPullWithThreeWayMerge(self):
"""Test clone, commit on remote and pull with merge, which resolves without conflicts."""
graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryDirectory() as localDirectory:
quitRepo = quit.git.Repository(localDirectory,
create=True,
origin=remote.path)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
index = quitRepo.index(remoteHead)
graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
index.add("graph2.nt", graph2Content)
index.add("graph2.nt.graph", "http://example2.org/\n")
author = Signature('QuitStoreTest', '*****@*****.**')
localCommitid = index.commit("from local", author.name,
author.email)
quitRepo._repository.checkout_tree(
quitRepo._repository.get(localCommitid))
self.assertEqual(
quitRepo.revision('HEAD').id, str(localCommitid))
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteQuitRepo = quit.git.Repository(remote.workdir)
index = remoteQuitRepo.index(remoteHead)
graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
index.add("graph.nt", graphContent)
remoteCommitid = index.commit("from remote", author.name,
author.email)
remoteQuitRepo._repository.checkout_tree(
remoteQuitRepo._repository.get(remoteCommitid))
quitRepo.pull(method="three-way")
self.assertNotEqual(
quitRepo.revision('HEAD').id, str(localCommitid))
self.assertNotEqual(
quitRepo.revision('HEAD').id, str(remoteCommitid))
# check if head has local and remote commit id as ancestor
self.assertListEqual([
parent.id for parent in quitRepo.revision('HEAD').parents
], [str(localCommitid),
str(remoteCommitid)])
# check if the merged commit contains all file contents
with open(path.join(localDirectory, "graph.nt")) as f:
self.assertEqual("".join(f.readlines()), graphContent)
with open(path.join(localDirectory, "graph2.nt")) as f:
self.assertEqual("".join(f.readlines()), graph2Content)
def testPullRepoClonedAndPullWithConflict(self):
"""Test clone, commit on remote and pull with conflict."""
graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryDirectory() as localDirectory:
quitRepo = quit.git.Repository(localDirectory,
create=True,
origin=remote.path)
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
index = quitRepo._repository.index
with open(path.join(localDirectory, "graph.nt"),
"a") as graphFile:
graphFile.write(
"<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
)
index.add("graph.nt")
index.write()
tree = index.write_tree()
author = Signature('QuitStoreTest', '*****@*****.**')
localCommitid = quitRepo._repository.create_commit(
'HEAD', author, author, "from local", tree, [remoteHead])
self.assertFalse(remote.is_empty)
self.assertFalse(quitRepo.is_empty)
index = remote.index
with open(path.join(remote.workdir, "graph.nt"),
"a") as graphFile:
graphFile.write(
"<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
)
index.add("graph.nt")
index.write()
tree = index.write_tree()
remoteCommitid = remote.create_commit('HEAD', author, author,
"from remote", tree,
[remoteHead])
remoteQuitRepo = quit.git.Repository(remote.workdir)
with self.assertRaises(QuitMergeConflict):
quitRepo.pull(method="context")
self.assertEqual(
quitRepo.revision('HEAD').id, str(localCommitid))
self.assertEqual(
remoteQuitRepo.revision('HEAD').id, str(remoteCommitid))
def testPullRepoFromNamedRemote(self):
"""Test if it is possible to pull from a remote repository, which is not called origin."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(False) as local:
local.remotes.create("upstream", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
remoteHead = remote.revparse_single('HEAD').hex
with self.assertRaises(RevisionNotFound):
quitRepo.revision('HEAD')
quitRepo.pull(remote_name='upstream', refspec="master")
self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)
self.assertFalse(remote.is_empty)
self.assertFalse(local.is_empty)
self.assertFalse(quitRepo.is_empty)
def testPullRepoFromNotConfiguredRemote(self):
"""Test if it is possible to pull from a remote repository, which is not called origin."""
graphContent = """
<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
with TemporaryRepositoryFactory().withGraph("http://example.org/",
graphContent) as remote:
with TemporaryRepository(False) as local:
local.remotes.create("origin", remote.path)
quitRepo = quit.git.Repository(local.workdir)
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
with self.assertRaises(RemoteNotFound):
quitRepo.pull(remote_name='upstream')
self.assertFalse(remote.is_empty)
self.assertTrue(local.is_empty)
self.assertTrue(quitRepo.is_empty)
def testPullRepoNoFFW(self):
"""TODO"""
pass
def testPullRepoFromMasterToDevelop(self):
"""TODO"""
pass
def testPullRepoFromDevelopToMaster(self):
"""TODO"""
pass
def testPullRepoFromRemoteTrackingBranch(self):
"""TODO"""
pass
def testRepositoryIsEmpty(self):
"""Test that adding data causes a new commit."""
self.addfile()
repo = quit.git.Repository(self.dir.name)
self.assertTrue(repo.is_empty)
self.createcommit()
repo = quit.git.Repository(self.dir.name)
self.assertFalse(repo.is_empty)
def testRepositoryIsBare(self):
"""Test if is_bare is currently done in init/clone tests."""
pass
def testNoGCConfiguration(self):
"""Test Garbage Collection configuration."""
quit.git.Repository(self.dir.name, garbageCollection=False)
with subprocess.Popen(["git", "config", "gc.auto"],
stdout=subprocess.PIPE,
cwd=self.dir.name) as getGCAuto:
stdout, stderr = getGCAuto.communicate()
response = stdout.decode("UTF-8").strip()
self.assertEqual(response, '')
def testGCConfiguration(self):
"""Test Garbage Collection configuration."""
quit.git.Repository(self.dir.name, garbageCollection=True)
with subprocess.Popen(["git", "config", "gc.auto"],
stdout=subprocess.PIPE,
cwd=self.dir.name) as getGCAuto:
stdout, stderr = getGCAuto.communicate()
response = stdout.decode("UTF-8").strip()
self.assertNotEqual(response, '')
self.assertEqual(response, '256')
def main():
from argparse import ArgumentParser
from os import path
import numpy as np
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('output_caffemodel')
parser.add_argument(
'-l',
'--load',
help=
'Load a pretrained model and rescale it [bias and type are not supported]'
)
parser.add_argument('-d',
'--data',
default=None,
help='Image list to use [default prototxt data]')
parser.add_argument('-b', '--bias', type=float, default=0.1, help='Bias')
parser.add_argument(
'-t',
'--type',
default='elwise',
help=
'Type: elwise, pca, zca, kmeans, rand (random input patches). Add fast_ to speed up the initialization, but you might lose in precision.'
)
parser.add_argument('-z',
action='store_true',
help='Zero all weights and reinitialize')
parser.add_argument('-cs', action='store_true', help='Correct for scaling')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-s',
type=float,
default=1.0,
help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs',
type=int,
default=16,
help='Batch size [only custom data "-d"]')
parser.add_argument('-nit',
type=int,
default=10,
help='Number of iterations')
parser.add_argument(
'--mem-limit',
type=int,
default=500,
help='How much memory should we use for the data buffer (MB)?')
parser.add_argument('--gpu',
type=int,
default=0,
help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
if args.data is not None:
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'" % args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i) + ' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name,
batch_size=args.bs,
new_width=model.input_dim[-1],
new_height=model.input_dim[-1],
transform_param=dict(
mean_value=[104, 117, 123],
scale=args.s),
ntop=2)
net.out = model(data=net.data, label=net.label)
else:
net.out = model()
n = netFromString('force_backward:true\n' + str(net.to_proto()),
caffe.TRAIN)
if args.load is not None:
n.copy_from(args.load)
# Rescale existing layers?
#if args.fix:
#magicFix(n, args.nit)
if args.z:
# Zero out all layers
for l in n.layers:
for b in l.blobs:
b.data[...] = 0
if any([
np.abs(l.blobs[0].data).sum() < 1e-10 for l in n.layers
if len(l.blobs) > 0
]):
print([
m for l, m in zip(n.layers, n._layer_names)
if len(l.blobs) > 0 and np.abs(l.blobs[0].data).sum() < 1e-10
])
magicInitialize(n,
args.bias,
NIT=args.nit,
type=args.type,
max_data=args.mem_limit * 1024 * 1024 / 4)
else:
print("Network already initialized, skipping magic init")
if args.cs:
# A simply helper function that lets you figure out which layers are not
# homogeneous
#print( estimateHomogenety(n) )
calibrateGradientRatio(n)
n.save(args.output_caffemodel)
def run_playbook(hosts, playbook, tags=[], private_key_file=private_key_file):
# initialize needed objects
loader = DataLoader()
options = Options(connection='ssh',
private_key_file=private_key_file,
module_path='',
forks=100,
become=True,
become_method='sudo',
become_user='******',
check=False,
tags=tags)
passwords = dict(vault_pass='')
results_callback = ResultCallback()
host_file = NamedTemporaryFile(delete=False)
host_file.write(b'[servers]\n')
for i, h in enumerate(hosts):
print(i, " : ", h)
host_file.write(bytes('{0} num={1}\n'.format(h, i), encoding='utf8'))
host_file.close()
# set inventory
inventory = InventoryManager(loader=loader, sources=host_file.name)
variable_manager = VariableManager(loader=loader, inventory=inventory)
# setup playbook executor, before the run
pbex = PlaybookExecutor(playbooks=[playbook],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords)
pbex._tqm._stdout_callback = results_callback
# run playbook and get stats
result = pbex.run()
stats = pbex._tqm._stats
outputs = {
0: 'deployment successful',
1: 'error occurred during deployment',
2: 'one or more hosts failed',
4: 'one or more hosts unreachable',
255: 'unknown error occurred during deployment'
}
run_success = True
hosts = sorted(stats.processed.keys())
for h in hosts:
t = stats.summarize(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
os.remove(host_file.name)
try:
out = outputs[result]
except KeyError:
out = 'unrecognised error code'
return result, out
def from_file_using_temporary_files(cls,
file,
format=None,
codec=None,
parameters=None,
start_second=None,
duration=None,
**kwargs):
orig_file = file
file, close_file = _fd_or_path_or_tempfile(file, 'rb', tempfile=False)
if format:
format = format.lower()
format = AUDIO_FILE_EXT_ALIASES.get(format, format)
def is_format(f):
f = f.lower()
if format == f:
return True
if isinstance(orig_file, basestring):
return orig_file.lower().endswith(".{0}".format(f))
if isinstance(orig_file, bytes):
return orig_file.lower().endswith(
(".{0}".format(f)).encode('utf8'))
return False
if is_format("wav"):
try:
obj = cls._from_safe_wav(file)
if close_file:
file.close()
if start_second is None and duration is None:
return obj
elif start_second is not None and duration is None:
return obj[start_second * 1000:]
elif start_second is None and duration is not None:
return obj[:duration * 1000]
else:
return obj[start_second * 1000:(start_second + duration) *
1000]
except:
file.seek(0)
elif is_format("raw") or is_format("pcm"):
sample_width = kwargs['sample_width']
frame_rate = kwargs['frame_rate']
channels = kwargs['channels']
metadata = {
'sample_width': sample_width,
'frame_rate': frame_rate,
'channels': channels,
'frame_width': channels * sample_width
}
obj = cls(data=file.read(), metadata=metadata)
if close_file:
file.close()
if start_second is None and duration is None:
return obj
elif start_second is not None and duration is None:
return obj[start_second * 1000:]
elif start_second is None and duration is not None:
return obj[:duration * 1000]
else:
return obj[start_second * 1000:(start_second + duration) *
1000]
input_file = NamedTemporaryFile(mode='wb', delete=False)
try:
input_file.write(file.read())
except (OSError):
input_file.flush()
input_file.close()
input_file = NamedTemporaryFile(mode='wb',
delete=False,
buffering=2**31 - 1)
if close_file:
file.close()
close_file = True
file = open(orig_file, buffering=2**13 - 1, mode='rb')
reader = file.read(2**31 - 1)
while reader:
input_file.write(reader)
reader = file.read(2**31 - 1)
input_file.flush()
if close_file:
file.close()
output = NamedTemporaryFile(mode="rb", delete=False)
conversion_command = [
cls.converter,
'-y', # always overwrite existing files
]
# If format is not defined
# ffmpeg/avconv will detect it automatically
if format:
conversion_command += ["-f", format]
if codec:
# force audio decoder
conversion_command += ["-acodec", codec]
conversion_command += [
"-i",
input_file.name, # input_file options (filename last)
"-vn", # Drop any video streams if there are any
"-f",
"wav" # output options (filename last)
]
if start_second is not None:
conversion_command += ["-ss", str(start_second)]
if duration is not None:
conversion_command += ["-t", str(duration)]
conversion_command += [output.name]
if parameters is not None:
# extend arguments with arbitrary set
conversion_command.extend(parameters)
log_conversion(conversion_command)
with open(os.devnull, 'rb') as devnull:
p = subprocess.Popen(conversion_command,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p_out, p_err = p.communicate()
log_subprocess_output(p_out)
log_subprocess_output(p_err)
try:
if p.returncode != 0:
raise CouldntDecodeError(
"Decoding failed. ffmpeg returned error code: {0}\n\nOutput from ffmpeg/avlib:\n\n{1}"
.format(p.returncode, p_err.decode(errors='ignore')))
obj = cls._from_safe_wav(output)
finally:
input_file.close()
output.close()
os.unlink(input_file.name)
os.unlink(output.name)
if start_second is None and duration is None:
return obj
elif start_second is not None and duration is None:
return obj[0:]
elif start_second is None and duration is not None:
return obj[:duration * 1000]
else:
return obj[0:duration * 1000]
class UcareCommonConfigFileTest(unittest.TestCase):
def setUp(self):
self.tmp_config_file = NamedTemporaryFile(mode='w+t', delete=False)
def tearDown(self):
conf.pub_key = None
conf.secret = None
conf.api_base = 'https://api.uploadcare.com/'
conf.upload_base = 'https://upload.uploadcare.com/'
conf.verify_api_ssl = True
conf.verify_upload_ssl = True
def test_use_pub_key_from_config_file(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'pub_key = demopublickey'
)
self.tmp_config_file.close()
main(arg_namespace('list_files'), [self.tmp_config_file.name])
self.assertEqual(conf.pub_key, 'demopublickey')
def test_redefine_pub_key_by_second_config_file(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'pub_key = demopublickey'
)
self.tmp_config_file.close()
second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False)
second_tmp_conf_file.write(
'[ucare]\n'
'pub_key = demopublickey_modified'
)
second_tmp_conf_file.close()
main(arg_namespace('list_files'),
[self.tmp_config_file.name, second_tmp_conf_file.name])
self.assertEqual(conf.pub_key, 'demopublickey_modified')
def test_use_available_pub_key_from_config_files(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'pub_key = demopublickey'
)
self.tmp_config_file.close()
second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False)
second_tmp_conf_file.write(
'[ucare]\n'
'secret = demosecretkey'
)
second_tmp_conf_file.close()
main(arg_namespace('list_files'),
[self.tmp_config_file.name, second_tmp_conf_file.name])
self.assertEqual(conf.pub_key, 'demopublickey')
def test_redefine_config_pub_key_by_args(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'pub_key = demopublickey'
)
self.tmp_config_file.close()
main(arg_namespace('--pub_key pub list_files'),
[self.tmp_config_file.name])
self.assertEqual(conf.pub_key, 'pub')
def test_load_verify_api_ssl_false_value_from_config(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'verify_api_ssl = false'
)
self.tmp_config_file.close()
main(arg_namespace('list_files'), [self.tmp_config_file.name])
self.assertFalse(conf.verify_api_ssl)
def test_load_verify_api_ssl_true_value_from_config(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'verify_api_ssl = true'
)
self.tmp_config_file.close()
main(arg_namespace('list_files'), [self.tmp_config_file.name])
self.assertTrue(conf.verify_api_ssl)
def test_redefine_config_verify_api_ssl_by_args(self, request):
request.return_value = MockListResponse()
self.tmp_config_file.write(
'[ucare]\n'
'verify_api_ssl = true'
)
self.tmp_config_file.close()
main(arg_namespace('--no_check_api_certificate list_files'),
[self.tmp_config_file.name])
self.assertFalse(conf.verify_api_ssl)
def run_fio(remote, config, rbd_test_dir):
"""
create fio config file with options based on above config
get the fio from github, generate binary, and use it to run on
the generated fio config file
"""
fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False)
fio_config.write('[global]\n')
if config.get('io-engine'):
ioengine=config['io-engine']
fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
else:
fio_config.write('ioengine=sync\n')
if config.get('bs'):
bs=config['bs']
fio_config.write('bs={bs}\n'.format(bs=bs))
else:
fio_config.write('bs=4k\n')
iodepth = config.get('io-depth', 2)
fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
if config.get('fio-io-size'):
size=config['fio-io-size']
fio_config.write('size={size}\n'.format(size=size))
else:
fio_config.write('size=100m\n')
fio_config.write('time_based\n')
if config.get('runtime'):
runtime=config['runtime']
fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
else:
fio_config.write('runtime=1800\n')
fio_config.write('allow_file_create=0\n')
image_size=10240
if config.get('image_size'):
image_size=config['image_size']
formats=[1,2]
features=[['layering'],['striping'],['exclusive-lock','object-map']]
fio_version='2.7'
if config.get('formats'):
formats=config['formats']
if config.get('features'):
features=config['features']
if config.get('fio-version'):
fio_version=config['fio-version']
# handle package required for ioengine, if any
sn=remote.shortname
ioengine_pkg = get_ioengine_package_name(ioengine, remote)
if ioengine_pkg:
install_package(ioengine_pkg, remote)
fio_config.write('norandommap\n')
if ioengine == 'rbd':
fio_config.write('clientname=admin\n')
fio_config.write('pool=rbd\n')
fio_config.write('invalidate=0\n')
elif ioengine == 'libaio':
fio_config.write('direct=1\n')
for frmt in formats:
for feature in features:
log.info("Creating rbd images on {sn}".format(sn=sn))
feature_name = '-'.join(feature)
rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
create_args=['rbd', 'create',
'--size', '{size}'.format(size=image_size),
'--image', rbd_name,
'--image-format', '{f}'.format(f=frmt)]
map(lambda x: create_args.extend(['--image-feature', x]), feature)
remote.run(args=create_args)
remote.run(args=['rbd', 'info', rbd_name])
if ioengine != 'rbd':
rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
if config.get('test-clone-io'):
log.info("Testing clones using fio")
remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
if config.get('rw'):
rw=config['rw']
fio_config.write('rw={rw}\n'.format(rw=rw))
else:
fio_config .write('rw=randrw\n')
fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
if config.get('test-clone-io'):
fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
fio_config.write('rw={rw}\n'.format(rw=rw))
fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
else:
if config.get('test-clone-io'):
log.info("Testing clones using fio")
remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
if config.get('rw'):
rw=config['rw']
fio_config.write('rw={rw}\n'.format(rw=rw))
else:
fio_config.write('rw=randrw\n')
fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
if config.get('test-clone-io'):
fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
fio_config.write('rw={rw}\n'.format(rw=rw))
fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
fio_config.close()
remote.put_file(fio_config.name,fio_config.name)
try:
log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
remote.run(args=['cd' , run.Raw(rbd_test_dir),
run.Raw(';'), 'wget' , fio , run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
run.Raw('cd fio-fio*'), 'configure', run.Raw(';') ,'make'])
remote.run(args=['ceph', '-s'])
remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
remote.run(args=['ceph', '-s'])
finally:
out=StringIO.StringIO()
remote.run(args=['rbd','showmapped', '--format=json'], stdout=out)
mapped_images = json.loads(out.getvalue())
if mapped_images:
log.info("Unmapping rbd images on {sn}".format(sn=sn))
for image in mapped_images.itervalues():
remote.run(args=['sudo', 'rbd', 'unmap', str(image['device'])])
log.info("Cleaning up fio install")
remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
if ioengine_pkg:
remove_package(ioengine_pkg, remote)
def gen_keys():
# good account key
account_key = NamedTemporaryFile()
Popen(["openssl", "genrsa", "-out", account_key.name, "2048"]).wait()
# weak 1024 bit key
weak_key = NamedTemporaryFile()
Popen(["openssl", "genrsa", "-out", weak_key.name, "1024"]).wait()
# good domain key
domain_key = NamedTemporaryFile()
domain_csr = NamedTemporaryFile()
Popen([
"openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout",
domain_key.name, "-subj", "/CN={0}".format(DOMAIN), "-out",
domain_csr.name
]).wait()
# subject alt-name domain
san_csr = NamedTemporaryFile()
san_conf = NamedTemporaryFile()
san_conf.write(open("/etc/ssl/openssl.cnf").read().encode("utf8"))
san_conf.write(
"\n[SAN]\nsubjectAltName=DNS:{0}\n".format(DOMAIN).encode("utf8"))
san_conf.seek(0)
Popen([
"openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
"/", "-reqexts", "SAN", "-config", san_conf.name, "-out", san_csr.name
]).wait()
# invalid domain csr
invalid_csr = NamedTemporaryFile()
Popen([
"openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
"/CN=\xC3\xA0\xC2\xB2\xC2\xA0_\xC3\xA0\xC2\xB2\xC2\xA0.com", "-out",
invalid_csr.name
]).wait()
# nonexistent domain csr
nonexistent_csr = NamedTemporaryFile()
Popen([
"openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
"/CN=404.{0}".format(DOMAIN), "-out", nonexistent_csr.name
]).wait()
#account-signed domain csr
account_csr = NamedTemporaryFile()
Popen([
"openssl", "req", "-new", "-sha256", "-key", account_key.name, "-subj",
"/CN={0}".format(DOMAIN), "-out", account_csr.name
]).wait()
return {
"account_key": account_key,
"weak_key": weak_key,
"domain_key": domain_key,
"domain_csr": domain_csr,
"san_csr": san_csr,
"invalid_csr": invalid_csr,
"nonexistent_csr": nonexistent_csr,
"account_csr": account_csr,
}
def write(data, path, html=None):
with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip:
if html is not None:
zip.writestr('index.html', html)
content = io.StringIO()
content.write('Manifest-Version: 1.0\n')
content.write('Created-By: jamovi\n')
content.write('Data-Archive-Version: 1.0.2\n')
content.write('jamovi-Archive-Version: 1.0\n')
zip.writestr('META-INF/MANIFEST.MF',
bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED)
content = None
fields = []
for column in data.dataset:
field = {}
field['name'] = column.name
field['columnType'] = ColumnType.stringify(column.column_type)
field['measureType'] = MeasureType.stringify(column.measure_type)
field['formula'] = column.formula
field['formulaMessage'] = column.formula_message
if column.measure_type == MeasureType.CONTINUOUS:
field['type'] = 'number'
else:
field['type'] = 'integer'
field['importName'] = column.import_name
fields.append(field)
metadata = {}
metadataset = {}
metadataset['rowCount'] = data.dataset.row_count
metadataset['columnCount'] = data.dataset.column_count
metadataset['fields'] = fields
metadata['dataSet'] = metadataset
zip.writestr('metadata.json', json.dumps(metadata),
zipfile.ZIP_DEFLATED)
metadata = None
xdata = {}
for column in data.dataset:
if column.has_levels:
xdata[column.name] = {'labels': column.levels}
zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED)
xdata = None
row_count = data.dataset.row_count
required_bytes = 0
for column in data.dataset:
if column.measure_type == MeasureType.CONTINUOUS:
required_bytes += (8 * row_count)
else:
required_bytes += (4 * row_count)
temp_file = NamedTemporaryFile(delete=False)
temp_file.truncate(required_bytes)
for column in data.dataset:
if column.measure_type == MeasureType.CONTINUOUS:
for i in range(0, row_count):
value = column.raw(i)
byts = struct.pack('<d', value)
temp_file.write(byts)
else:
for i in range(0, row_count):
value = column.raw(i)
byts = struct.pack('<i', value)
temp_file.write(byts)
temp_file.close()
zip.write(temp_file.name, 'data.bin')
os.remove(temp_file.name)
resources = []
for analysis in data.analyses:
if analysis.has_results is False:
continue
analysis_dir = '{:02} {}/analysis'.format(analysis.id,
analysis.name)
zip.writestr(analysis_dir, analysis.serialize(),
zipfile.ZIP_DEFLATED)
resources += analysis.resources
for rel_path in resources:
abs_path = os.path.join(data.instance_path, rel_path)
zip.write(abs_path, rel_path)
CN_FILE = abspath(expanduser(CONFIG_DICT['contact_network_file']))
CONFIG_DICT['contact_network_file'] = '/FAVITES_MOUNT/%s' % CN_FILE.split('/')[-1]
TN_FILE = None
if 'transmission_network_file' in CONFIG_DICT:
TN_FILE = abspath(expanduser(CONFIG_DICT['transmission_network_file']))
CONFIG_DICT['transmission_network_file'] = '/FAVITES_MOUNT/%s' % TN_FILE.split('/')[-1]
TREE_FILE = None
if 'tree_file' in CONFIG_DICT:
TREE_FILE = abspath(expanduser(CONFIG_DICT['tree_file']))
CONFIG_DICT['tree_file'] = '/FAVITES_MOUNT/%s' % TREE_FILE.split('/')[-1]
ERRORFREE_SEQ_FILE = None
if 'errorfree_sequence_file' in CONFIG_DICT:
ERRORFREE_SEQ_FILE = abspath(expanduser(CONFIG_DICT['errorfree_sequence_file']))
CONFIG_DICT['errorfree_sequence_file'] = '/FAVITES_MOUNT/%s' % ERRORFREE_SEQ_FILE.split('/')[-1]
TMP_CONFIG = NamedTemporaryFile('w')
TMP_CONFIG.write(str(CONFIG_DICT))
TMP_CONFIG.flush()
# pull the newest versioned Docker image (if applicable)
if args.update is None:
version = None
try:
o = check_output(['docker','images']).decode().splitlines()
for l in o:
if l.startswith(DOCKER_IMAGE):
version = '%s:%s' % (DOCKER_IMAGE,l.split()[1]); break
except CalledProcessError as e:
raise RuntimeError("docker images command failed\n%s"%e.output)
if version is None:
args.update = []
if args.update is not None:
class Git:
def __init__(self, git_repo, repo_dir, pkey=None):
self.git_repo = git_repo
self.repo_dir = repo_dir
self.repo = None
self.pkey = pkey
self.fd = None
self.env = {}
def archive(self, filepath, commit):
with open(filepath, 'wb') as f:
self.repo.archive(f, commit)
def fetch_branches_tags(self):
self.fetch()
branches, tags = {}, {}
for ref in self.repo.references:
if isinstance(ref, RemoteReference):
if ref.remote_head != 'HEAD':
branches[ref.remote_head] = self._get_commits(
f'origin/{ref.remote_head}')
elif isinstance(ref, TagReference):
tags[ref.name] = {
'id': ref.tag.hexsha,
'author': ref.tag.tagger.name,
'date': self._format_date(ref.tag.tagged_date),
'message': ref.tag.message.strip()
} if ref.tag else {
'id': ref.commit.binsha.hex(),
'author': ref.commit.author.name,
'date': self._format_date(ref.commit.authored_date),
'message': ref.commit.message.strip()
}
tags = sorted(tags.items(), key=lambda x: x[1]['date'], reverse=True)
return branches, dict(tags)
def fetch(self):
try:
self.repo.remotes.origin.fetch(f=True, p=True, P=True)
except GitCommandError as e:
if self.env:
self.repo.remotes.origin.fetch(env=self.env,
f=True,
p=True,
P=True)
else:
raise e
def _get_repo(self):
if os.path.exists(self.repo_dir):
try:
return Repo(self.repo_dir)
except InvalidGitRepositoryError:
if os.path.isdir(self.repo_dir):
shutil.rmtree(self.repo_dir)
else:
os.remove(self.repo_dir)
try:
repo = Repo.clone_from(self.git_repo, self.repo_dir)
except GitCommandError as e:
if self.env:
repo = Repo.clone_from(self.git_repo,
self.repo_dir,
env=self.env)
else:
raise e
return repo
def _get_commits(self, branch, count=10):
commits = []
for commit in self.repo.iter_commits(branch):
if len(commits) == count:
break
commits.append({
'id': commit.hexsha,
'author': commit.author.name,
'date': self._format_date(commit.committed_date),
'message': commit.message.strip()
})
return commits
def _format_date(self, timestamp):
if isinstance(timestamp, int):
date = datetime.fromtimestamp(timestamp)
return date.strftime('%Y-%m-%d %H:%M')
return timestamp
def __enter__(self):
if self.pkey:
self.fd = NamedTemporaryFile()
self.fd.write(self.pkey.encode())
self.fd.flush()
self.env = {'GIT_SSH_COMMAND': f'ssh -i {self.fd.name}'}
self.repo = self._get_repo()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.fd:
self.fd.close()
jobLog = open('patch_tested.txt', 'w+')
if jobLogHistory:
if len(jobLogHistory) > options.history:
jobLogHistory = [jobLogHistory[0]] \
+ jobLogHistory[len(jobLogHistory)
- options.history:]
for jobHistoryRecord in jobLogHistory:
jobLog.write(jobHistoryRecord + '\n')
else:
jobLog.write('TESTED ISSUES\n')
jobLog.flush()
rssData = httpGet(options.jiraFilter, False, options.jiraUser,
options.jiraPassword)
tempFile.write(rssData)
tempFile.flush()
for (key, attachment) in parseJiraData(tempFile.name).items():
(project, issue) = key
if jenkinsUrlOverrides.has_key(project):
url = jenkinsUrlOverrides[project]
else:
url = options.jenkinsUrl
jenkinsUrlTemplate = url + '/job/' \
+ options.jenkinsJobTemplate \
+ '/buildWithParameters?' + tokenFrag \
+ '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}'
urlArgs = {
'project': project,
def main(apiurl, opts, argv):
repo = argv[0]
arch = argv[1]
build_descr = argv[2]
xp = []
build_root = None
cache_dir = None
build_uid = ''
vm_type = config['build-type']
vm_telnet = None
build_descr = os.path.abspath(build_descr)
build_type = os.path.splitext(build_descr)[1][1:]
if os.path.basename(build_descr) == 'PKGBUILD':
build_type = 'arch'
if os.path.basename(build_descr) == 'build.collax':
build_type = 'collax'
if build_type not in ['spec', 'dsc', 'kiwi', 'arch', 'collax', 'livebuild']:
raise oscerr.WrongArgs(
'Unknown build type: \'%s\'. Build description should end in .spec, .dsc, .kiwi or .livebuild.' \
% build_type)
if not os.path.isfile(build_descr):
raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr)
buildargs = []
if not opts.userootforbuild:
buildargs.append('--norootforbuild')
if opts.clean:
buildargs.append('--clean')
if opts.noinit:
buildargs.append('--noinit')
if opts.nochecks:
buildargs.append('--no-checks')
if not opts.no_changelog:
buildargs.append('--changelog')
if opts.root:
build_root = opts.root
if opts.target:
buildargs.append('--target=%s' % opts.target)
if opts.threads:
buildargs.append('--threads=%s' % opts.threads)
if opts.jobs:
buildargs.append('--jobs=%s' % opts.jobs)
elif config['build-jobs'] > 1:
buildargs.append('--jobs=%s' % config['build-jobs'])
if opts.icecream or config['icecream'] != '0':
if opts.icecream:
num = opts.icecream
else:
num = config['icecream']
if int(num) > 0:
buildargs.append('--icecream=%s' % num)
xp.append('icecream')
xp.append('gcc-c++')
if opts.ccache:
buildargs.append('--ccache')
xp.append('ccache')
if opts.linksources:
buildargs.append('--linksources')
if opts.baselibs:
buildargs.append('--baselibs')
if opts.debuginfo:
buildargs.append('--debug')
if opts._with:
for o in opts._with:
buildargs.append('--with=%s' % o)
if opts.without:
for o in opts.without:
buildargs.append('--without=%s' % o)
if opts.define:
for o in opts.define:
buildargs.append('--define=%s' % o)
if config['build-uid']:
build_uid = config['build-uid']
if opts.build_uid:
build_uid = opts.build_uid
if build_uid:
buildidre = re.compile('^[0-9]{1,5}:[0-9]{1,5}$')
if build_uid == 'caller':
buildargs.append('--uid=%s:%s' % (os.getuid(), os.getgid()))
elif buildidre.match(build_uid):
buildargs.append('--uid=%s' % build_uid)
else:
print('Error: build-uid arg must be 2 colon separated numerics: "uid:gid" or "caller"', file=sys.stderr)
return 1
if opts.vm_type:
vm_type = opts.vm_type
if opts.vm_telnet:
vm_telnet = opts.vm_telnet
if opts.alternative_project:
prj = opts.alternative_project
pac = '_repository'
else:
prj = store_read_project(os.curdir)
if opts.local_package:
pac = '_repository'
else:
pac = store_read_package(os.curdir)
if opts.shell:
buildargs.append("--shell")
orig_build_root = config['build-root']
# make it possible to override configuration of the rc file
for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']:
val = os.getenv(var)
if val:
if var.startswith('OSC_'): var = var[4:]
var = var.lower().replace('_', '-')
if var in config:
print('Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val))
config[var] = val
pacname = pac
if pacname == '_repository':
if not opts.local_package:
try:
pacname = store_read_package(os.curdir)
except oscerr.NoWorkingCopy:
opts.local_package = True
if opts.local_package:
pacname = os.path.splitext(build_descr)[0]
apihost = urlsplit(apiurl)[1]
if not build_root:
build_root = config['build-root']
if build_root == orig_build_root:
# ENV var was not set
build_root = config['api_host_options'][apiurl].get('build-root', build_root)
try:
build_root = build_root % {'repo': repo, 'arch': arch,
'project': prj, 'package': pacname, 'apihost': apihost}
except:
pass
cache_dir = config['packagecachedir'] % {'apihost': apihost}
extra_pkgs = []
if not opts.extra_pkgs:
extra_pkgs = config['extra-pkgs']
elif opts.extra_pkgs != ['']:
extra_pkgs = opts.extra_pkgs
if xp:
extra_pkgs += xp
prefer_pkgs = {}
build_descr_data = open(build_descr).read()
# XXX: dirty hack but there's no api to provide custom defines
if opts.without:
s = ''
for i in opts.without:
s += "%%define _without_%s 1\n" % i
build_descr_data = s + build_descr_data
if opts._with:
s = ''
for i in opts._with:
s += "%%define _with_%s 1\n" % i
build_descr_data = s + build_descr_data
if opts.define:
s = ''
for i in opts.define:
s += "%%define %s\n" % i
build_descr_data = s + build_descr_data
cpiodata = None
servicefile = os.path.join(os.path.dirname(build_descr), "_service")
if not os.path.isfile(servicefile):
servicefile = os.path.join(os.path.dirname(build_descr), "_service")
if not os.path.isfile(servicefile):
servicefile = None
else:
print('Using local _service file')
buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv." + repo + "." + arch)
if not os.path.isfile(buildenvfile):
buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv")
if not os.path.isfile(buildenvfile):
buildenvfile = None
else:
print('Using local buildenv file: %s' % os.path.basename(buildenvfile))
if buildenvfile or servicefile:
from .util import cpio
if not cpiodata:
cpiodata = cpio.CpioWrite()
if opts.prefer_pkgs:
print('Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs))
from .util import cpio
if not cpiodata:
cpiodata = cpio.CpioWrite()
prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type, cpiodata)
if cpiodata:
cpiodata.add(os.path.basename(build_descr), build_descr_data)
# buildenv must come last for compatibility reasons...
if buildenvfile:
cpiodata.add("buildenv", open(buildenvfile).read())
if servicefile:
cpiodata.add("_service", open(servicefile).read())
build_descr_data = cpiodata.get()
# special handling for overlay and rsync-src/dest
specialcmdopts = []
if opts.rsyncsrc or opts.rsyncdest :
if not opts.rsyncsrc or not opts.rsyncdest:
raise oscerr.WrongOptions('When using --rsync-{src,dest} both parameters have to be specified.')
myrsyncsrc = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.rsyncsrc)))
if not os.path.isdir(myrsyncsrc):
raise oscerr.WrongOptions('--rsync-src %s is no valid directory!' % opts.rsyncsrc)
# can't check destination - its in the target chroot ;) - but we can check for sanity
myrsyncdest = os.path.expandvars(opts.rsyncdest)
if not os.path.isabs(myrsyncdest):
raise oscerr.WrongOptions('--rsync-dest %s is no absolute path (starting with \'/\')!' % opts.rsyncdest)
specialcmdopts = ['--rsync-src='+myrsyncsrc, '--rsync-dest='+myrsyncdest]
if opts.overlay:
myoverlay = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.overlay)))
if not os.path.isdir(myoverlay):
raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)
specialcmdopts += ['--overlay='+myoverlay]
bi_file = None
bc_file = None
bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
bc_filename = '_buildconfig-%s-%s' % (repo, arch)
if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
elif not os.access('.', os.W_OK):
bi_file = NamedTemporaryFile(prefix=bi_filename)
bi_filename = bi_file.name
bc_file = NamedTemporaryFile(prefix=bc_filename)
bc_filename = bc_file.name
else:
bi_filename = os.path.abspath(bi_filename)
bc_filename = os.path.abspath(bc_filename)
try:
if opts.noinit:
if not os.path.isfile(bi_filename):
raise oscerr.WrongOptions('--noinit is not possible, no local buildinfo file')
print('Use local \'%s\' file as buildinfo' % bi_filename)
if not os.path.isfile(bc_filename):
raise oscerr.WrongOptions('--noinit is not possible, no local buildconfig file')
print('Use local \'%s\' file as buildconfig' % bc_filename)
elif opts.offline:
if not os.path.isfile(bi_filename):
raise oscerr.WrongOptions('--offline is not possible, no local buildinfo file')
print('Use local \'%s\' file as buildinfo' % bi_filename)
if not os.path.isfile(bc_filename):
raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')
else:
print('Getting buildinfo from server and store to %s' % bi_filename)
bi_text = ''.join(get_buildinfo(apiurl,
prj,
pac,
repo,
arch,
specfile=build_descr_data,
addlist=extra_pkgs))
if not bi_file:
bi_file = open(bi_filename, 'w')
# maybe we should check for errors before saving the file
bi_file.write(bi_text)
bi_file.flush()
print('Getting buildconfig from server and store to %s' % bc_filename)
bc = get_buildconfig(apiurl, prj, repo)
if not bc_file:
bc_file = open(bc_filename, 'w')
bc_file.write(bc)
bc_file.flush()
except HTTPError as e:
if e.code == 404:
# check what caused the 404
if meta_exists(metatype='prj', path_args=(quote_plus(prj), ),
template_args=None, create_new=False, apiurl=apiurl):
pkg_meta_e = None
try:
# take care, not to run into double trouble.
pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj),
quote_plus(pac)), template_args=None, create_new=False,
apiurl=apiurl)
except:
pass
if pkg_meta_e:
print('ERROR: Either wrong repo/arch as parameter or a parse error of .spec/.dsc/.kiwi file due to syntax error', file=sys.stderr)
else:
print('The package \'%s\' does not exist - please ' \
'rerun with \'--local-package\'' % pac, file=sys.stderr)
else:
print('The project \'%s\' does not exist - please ' \
'rerun with \'--alternative-project <alternative_project>\'' % prj, file=sys.stderr)
sys.exit(1)
else:
raise
bi = Buildinfo(bi_filename, apiurl, build_type, list(prefer_pkgs.keys()))
if bi.debuginfo and not (opts.disable_debuginfo or '--debug' in buildargs):
buildargs.append('--debug')
if opts.release:
bi.release = opts.release
if bi.release:
buildargs.append('--release=%s' % bi.release)
# real arch of this machine
# vs.
# arch we are supposed to build for
if bi.hostarch != None:
if hostarch != bi.hostarch and not bi.hostarch in can_also_build.get(hostarch, []):
print('Error: hostarch \'%s\' is required.' % (bi.hostarch), file=sys.stderr)
return 1
elif hostarch != bi.buildarch:
if not bi.buildarch in can_also_build.get(hostarch, []):
# OBSOLETE: qemu_can_build should not be needed anymore since OBS 2.3
if vm_type != "emulator" and not bi.buildarch in qemu_can_build:
print('Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch), file=sys.stderr)
return 1
print('WARNING: It is guessed to build on hostarch \'%s\' for \'%s\' via QEMU.' % (hostarch, bi.buildarch), file=sys.stderr)
rpmlist_prefers = []
if prefer_pkgs:
print('Evaluating preferred packages')
for name, path in prefer_pkgs.items():
if bi.has_dep(name):
# We remove a preferred package from the buildinfo, so that the
# fetcher doesn't take care about them.
# Instead, we put it in a list which is appended to the rpmlist later.
# At the same time, this will make sure that these packages are
# not verified.
bi.remove_dep(name)
rpmlist_prefers.append((name, path))
print(' - %s (%s)' % (name, path))
print('Updating cache of required packages')
urllist = []
if not opts.download_api_only:
# transform 'url1, url2, url3' form into a list
if 'urllist' in config:
if isinstance(config['urllist'], str):
re_clist = re.compile('[, ]+')
urllist = [ i.strip() for i in re_clist.split(config['urllist'].strip()) ]
else:
urllist = config['urllist']
# OBS 1.5 and before has no downloadurl defined in buildinfo
if bi.downloadurl:
urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')
if opts.disable_cpio_bulk_download:
urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )
fetcher = Fetcher(cache_dir,
urllist = urllist,
api_host_options = config['api_host_options'],
offline = opts.noinit or opts.offline,
http_debug = config['http_debug'],
enable_cpio = not opts.disable_cpio_bulk_download,
cookiejar=cookiejar)
if not opts.trust_all_projects:
# implicitly trust the project we are building for
check_trusted_projects(apiurl, [ i for i in bi.projects.keys() if not i == prj ])
# now update the package cache
fetcher.run(bi)
old_pkg_dir = None
if opts.oldpackages:
old_pkg_dir = opts.oldpackages
if not old_pkg_dir.startswith('/') and not opts.offline:
data = [ prj, pacname, repo, arch]
if old_pkg_dir == '_link':
p = osc.core.findpacs(os.curdir)[0]
if not p.islink():
raise oscerr.WrongOptions('package is not a link')
data[0] = p.linkinfo.project
data[1] = p.linkinfo.package
repos = osc.core.get_repositories_of_project(apiurl, data[0])
# hack for links to e.g. Factory
if not data[2] in repos and 'standard' in repos:
data[2] = 'standard'
elif old_pkg_dir != '' and old_pkg_dir != '_self':
a = old_pkg_dir.split('/')
for i in range(0, len(a)):
data[i] = a[i]
destdir = os.path.join(cache_dir, data[0], data[2], data[3])
old_pkg_dir = None
try:
print("Downloading previous build from %s ..." % '/'.join(data))
binaries = get_binarylist(apiurl, data[0], data[2], data[3], package=data[1], verbose=True)
except Exception as e:
print("Error: failed to get binaries: %s" % str(e))
binaries = []
if binaries:
class mytmpdir:
""" temporary directory that removes itself"""
def __init__(self, *args, **kwargs):
self.name = mkdtemp(*args, **kwargs)
_rmtree = staticmethod(shutil.rmtree)
def cleanup(self):
self._rmtree(self.name)
def __del__(self):
self.cleanup()
def __exit__(self):
self.cleanup()
def __str__(self):
return self.name
old_pkg_dir = mytmpdir(prefix='.build.oldpackages', dir=os.path.abspath(os.curdir))
if not os.path.exists(destdir):
os.makedirs(destdir)
for i in binaries:
fname = os.path.join(destdir, i.name)
os.symlink(fname, os.path.join(str(old_pkg_dir), i.name))
if os.path.exists(fname):
st = os.stat(fname)
if st.st_mtime == i.mtime and st.st_size == i.size:
continue
get_binary_file(apiurl,
data[0],
data[2], data[3],
i.name,
package = data[1],
target_filename = fname,
target_mtime = i.mtime,
progress_meter = True)
if old_pkg_dir != None:
buildargs.append('--oldpackages=%s' % old_pkg_dir)
# Make packages from buildinfo available as repos for kiwi
if build_type == 'kiwi':
if os.path.exists('repos'):
shutil.rmtree('repos')
os.mkdir('repos')
for i in bi.deps:
if not i.extproject:
# remove
bi.deps.remove(i)
continue
# project
pdir = str(i.extproject).replace(':/', ':')
# repo
rdir = str(i.extrepository).replace(':/', ':')
# arch
adir = i.repoarch
# project/repo
prdir = "repos/"+pdir+"/"+rdir
# project/repo/arch
pradir = prdir+"/"+adir
# source fullfilename
sffn = i.fullfilename
filename = sffn.split("/")[-1]
# target fullfilename
tffn = pradir+"/"+filename
if not os.path.exists(os.path.join(pradir)):
os.makedirs(os.path.join(pradir))
if not os.path.exists(tffn):
print("Using package: "+sffn)
if opts.linksources:
os.link(sffn, tffn)
else:
os.symlink(sffn, tffn)
if prefer_pkgs:
for name, path in prefer_pkgs.items():
if name == filename:
print("Using prefered package: " + path + "/" + filename)
os.unlink(tffn)
if opts.linksources:
os.link(path + "/" + filename, tffn)
else:
os.symlink(path + "/" + filename, tffn)
# Is a obsrepositories tag used?
try:
tree = ET.parse(build_descr)
except:
print('could not parse the kiwi file:', file=sys.stderr)
print(open(build_descr).read(), file=sys.stderr)
sys.exit(1)
root = tree.getroot()
# product
for xml in root.findall('instsource'):
if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/':
print("obsrepositories:/ for product builds is not yet supported in osc!")
sys.exit(1)
# appliance
expand_obsrepos=None
for xml in root.findall('repository'):
if xml.find('source').get('path') == 'obsrepositories:/':
expand_obsrepos=True
if expand_obsrepos:
buildargs.append('--kiwi-parameter')
buildargs.append('--ignore-repos')
for xml in root.findall('repository'):
if xml.find('source').get('path') == 'obsrepositories:/':
for path in bi.pathes:
if not os.path.isdir("repos/"+path):
continue
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repo')
buildargs.append('--kiwi-parameter')
buildargs.append("repos/"+path)
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repotype')
buildargs.append('--kiwi-parameter')
buildargs.append('rpm-md')
if xml.get('priority'):
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repoprio='+xml.get('priority'))
else:
m = re.match(r"obs://[^/]+/([^/]+)/(\S+)", xml.find('source').get('path'))
if not m:
# short path without obs instance name
m = re.match(r"obs://([^/]+)/(.+)", xml.find('source').get('path'))
project=m.group(1).replace(":", ":/")
repo=m.group(2)
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repo')
buildargs.append('--kiwi-parameter')
buildargs.append("repos/"+project+"/"+repo)
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repotype')
buildargs.append('--kiwi-parameter')
buildargs.append('rpm-md')
if xml.get('priority'):
buildargs.append('--kiwi-parameter')
buildargs.append('--add-repopriority='+xml.get('priority'))
if vm_type == "xen" or vm_type == "kvm" or vm_type == "lxc":
print('Skipping verification of package signatures due to secure VM build')
elif bi.pacsuffix == 'rpm':
if opts.no_verify:
print('Skipping verification of package signatures')
else:
print('Verifying integrity of cached packages')
verify_pacs(bi)
elif bi.pacsuffix == 'deb':
if opts.no_verify or opts.noinit:
print('Skipping verification of package signatures')
else:
print('WARNING: deb packages get not verified, they can compromise your system !')
else:
print('WARNING: unknown packages get not verified, they can compromise your system !')
for i in bi.deps:
if i.hdrmd5:
from .util import packagequery
hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename)
if not hdrmd5:
print("Error: cannot get hdrmd5 for %s" % i.fullfilename)
sys.exit(1)
if hdrmd5 != i.hdrmd5:
print("Error: hdrmd5 mismatch for %s: %s != %s" % (i.fullfilename, hdrmd5, i.hdrmd5))
sys.exit(1)
print('Writing build configuration')
if build_type == 'kiwi':
rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ]
else:
rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps ]
rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ]
rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n')
rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n')
rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n')
if build_type != 'kiwi' and bi.noinstall_list:
rpmlist.append('noinstall: ' + ' '.join(bi.noinstall_list) + '\n')
if build_type != 'kiwi' and bi.installonly_list:
rpmlist.append('installonly: ' + ' '.join(bi.installonly_list) + '\n')
rpmlist_file = NamedTemporaryFile(prefix='rpmlist.')
rpmlist_filename = rpmlist_file.name
rpmlist_file.writelines(rpmlist)
rpmlist_file.flush()
subst = { 'repo': repo, 'arch': arch, 'project' : prj, 'package' : pacname }
vm_options = []
# XXX check if build-device present
my_build_device = ''
if config['build-device']:
my_build_device = config['build-device'] % subst
else:
# obs worker uses /root here but that collides with the
# /root directory if the build root was used without vm
# before
my_build_device = build_root + '/img'
need_root = True
if vm_type:
if config['build-swap']:
my_build_swap = config['build-swap'] % subst
else:
my_build_swap = build_root + '/swap'
vm_options = [ '--vm-type=%s' % vm_type ]
if vm_telnet:
vm_options += [ '--vm-telnet=' + vm_telnet ]
if config['build-memory']:
vm_options += [ '--memory=' + config['build-memory'] ]
if vm_type != 'lxc':
vm_options += [ '--vm-disk=' + my_build_device ]
vm_options += [ '--vm-swap=' + my_build_swap ]
vm_options += [ '--logfile=%s/.build.log' % build_root ]
if vm_type == 'kvm':
if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK):
# so let's hope there's also an fstab entry
need_root = False
if config['build-kernel']:
vm_options += [ '--vm-kernel=' + config['build-kernel'] ]
if config['build-initrd']:
vm_options += [ '--vm-initrd=' + config['build-initrd'] ]
build_root += '/.mount'
if config['build-memory']:
vm_options += [ '--memory=' + config['build-memory'] ]
if config['build-vmdisk-rootsize']:
vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ]
if config['build-vmdisk-swapsize']:
vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ]
if config['build-vmdisk-filesystem']:
vm_options += [ '--vmdisk-filesystem=' + config['build-vmdisk-filesystem'] ]
if config['build-vm-user']:
vm_options += [ '--vm-user='******'build-vm-user'] ]
if opts.preload:
print("Preload done for selected repo/arch.")
sys.exit(0)
print('Running build')
cmd = [ config['build-cmd'], '--root='+build_root,
'--rpmlist='+rpmlist_filename,
'--dist='+bc_filename,
'--arch='+bi.buildarch ]
cmd += specialcmdopts + vm_options + buildargs
cmd += [ build_descr ]
if need_root:
sucmd = config['su-wrapper'].split()
if sucmd[0] == 'su':
if sucmd[-1] == '-c':
sucmd.pop()
cmd = sucmd + ['-s', cmd[0], 'root', '--' ] + cmd[1:]
else:
cmd = sucmd + cmd
# change personality, if needed
if hostarch != bi.buildarch and bi.buildarch in change_personality:
cmd = [ change_personality[bi.buildarch] ] + cmd
try:
rc = run_external(cmd[0], *cmd[1:])
if rc:
print()
print('The buildroot was:', build_root)
sys.exit(rc)
except KeyboardInterrupt as i:
print("keyboard interrupt, killing build ...")
cmd.append('--kill')
run_external(cmd[0], *cmd[1:])
raise i
pacdir = os.path.join(build_root, '.build.packages')
if os.path.islink(pacdir):
pacdir = os.readlink(pacdir)
pacdir = os.path.join(build_root, pacdir)
if os.path.exists(pacdir):
(s_built, b_built) = get_built_files(pacdir, bi.buildtype)
print()
if s_built: print(s_built)
print()
print(b_built)
if opts.keep_pkgs:
for i in b_built.splitlines() + s_built.splitlines():
shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i)))
if bi_file:
bi_file.close()
if bc_file:
bc_file.close()
rpmlist_file.close()
def create_router(transport):
print("\033[1A\033[K" + "Creating shaper for {}".format(transport))
if transport == "bus":
routing_type = {
"weights": {"motorway": 1.5, "trunk": 1.5, "primary": 1.4, "secondary": 1.3, "tertiary": 1.3,
"unclassified": 1, "residential": 0.6, "living_street": 0.6, "track": 0.3, "service": 0.5},
"access": ["access", "vehicle", "motor_vehicle", "psv", "bus", "routing:ztm"],
"name": "bus"
}
elif transport == "train":
routing_type = "train"
elif transport == "tram":
routing_type = "tram"
else:
raise ValueError("Invalid transport type {} for Shaper".format(transport))
if transport in {"train", "tram"}:
request = requests.get("https://mkuran.pl/feed/ztm/ztm-km-rail-shapes.osm", stream=True)
else:
# Overpass Query:
query = "\n".join([
"[bbox:51.9144,20.4438,52.5007,21.4844][out:xml];"
"("
' way["highway"="motorway"];'
' way["highway"="motorway_link"];'
' way["highway"="trunk"];'
' way["highway"="trunk_link"];'
' way["highway"="primary"];'
' way["highway"="primary_link"];'
' way["highway"="secondary"];'
' way["highway"="secondary_link"];'
' way["highway"="tertiary"];'
' way["highway"="tertiary_link"];'
' way["highway"="unclassified"];'
' way["highway"="minor"];'
' way["highway"="residential"];'
' way["highway"="living_street"];'
' way["highway"="service"];'
');'
'way._(poly:"52.4455 20.6858 52.376 20.6872 52.3533 20.7868 52.2929 20.726 52.2694 20.6724 52.2740 20.4465 52.2599 20.4438 52.2481 20.5832 52.2538 20.681 52.1865 20.6786 52.1859 20.7129 52.1465 20.7895 52.0966 20.783 52.0632 20.7222 52.0151 20.7617 51.9873 20.9351 51.9269 20.9509 51.9144 21.0226 51.9322 21.1987 51.9569 21.2472 52.0463 21.2368 52.1316 21.4844 52.1429 21.4404 52.2130 21.3814 52.2622 21.3141 52.2652 21.1977 52.3038 21.173 52.3063 21.2925 52.3659 21.3515 52.3829 21.3001 52.4221 21.1929 52.4898 21.1421");'
'>->.n;'
'<->.r;'
'(._;.n;.r;);'
'out;'
])
request = requests.get(
"https://overpass-api.de/api/interpreter/",
params={"data": query},
stream=True,
)
temp_xml = NamedTemporaryFile(delete=False)
for chunk in request.iter_content(chunk_size=1024):
temp_xml.write(chunk)
request.close()
temp_xml.seek(0)
router = pyroutelib3.Router(routing_type, temp_xml.name)
temp_xml.close()
return router
class ParallelBlasterTests(TestCase):
def setUp(self):
""" """
self.files_to_remove = []
self.dirs_to_remove = []
tmp_dir = get_qiime_temp_dir()
self.test_out = get_tmp_filename(
tmp_dir=tmp_dir,
prefix='qiime_parallel_blaster_tests_',
suffix='',
result_constructor=str)
self.dirs_to_remove.append(self.test_out)
create_dir(self.test_out)
self.tmp_seq_filepath = get_tmp_filename(
tmp_dir=self.test_out,
prefix='qiime_parallel_blaster_tests_input',
suffix='.fasta')
seq_file = open(self.tmp_seq_filepath, 'w')
seq_file.write(blast_test_seqs)
seq_file.close()
self.files_to_remove.append(self.tmp_seq_filepath)
self.reference_seqs_file = NamedTemporaryFile(
prefix='qiime_parallel_blaster_tests_ref_seqs',
suffix='.fasta',
dir=tmp_dir)
self.reference_seqs_file.write(blast_ref_seqs)
self.reference_seqs_file.seek(0)
initiate_timeout(60)
def tearDown(self):
""" """
disable_timeout()
remove_files(self.files_to_remove)
# remove directories last, so we don't get errors
# trying to remove files which may be in the directories
for d in self.dirs_to_remove:
if exists(d):
rmtree(d)
def test_parallel_blaster(self):
"""Test ParallelBlaster functions as expected."""
params = {
'refseqs_path': self.reference_seqs_file.name,
'disable_low_complexity_filter': False,
'e_value': 0.001,
'num_hits': 1,
'word_size': 30,
'suppress_format_blastdb': False,
'blastmat_dir': None
}
app = ParallelBlaster()
r = app(self.tmp_seq_filepath,
self.test_out,
params,
job_prefix='BLASTTEST',
poll_directly=True,
suppress_submit_jobs=False)
# Basic sanity checks: we should get two blast hits (lines). We ignore
# all of the comments in the file. Each line should have 12 fields
# separated by tabs.
results = [
line for line in open(
glob(join(self.test_out, '*_blast_out.txt'))[0], 'U')
if not line.startswith('#')
]
self.assertEqual(len(results), 2)
self.assertEqual(len(results[0].split('\t')), 12)
self.assertEqual(len(results[1].split('\t')), 12)
regenerate = '--regenerate-test' in sys.argv[1:]
for dvd in False,True:
if regenerate:
f = open(test_template(dvd=dvd), 'w')
else:
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile(mode='w')
f.write(
re.sub('(?<!\r)\n', '\r\n',
generate(
dvd=dvd,
version=r'${NORMALIZED_VERSION}',
human_version=r'%(human_version)s',
sections='%(sections)s',
architecture='32'
)))
f.flush()
if not regenerate:
print subprocess.Popen(
['diff', '-wdu', test_template(dvd=dvd), f.name]).communicate()[0]
if not dvd:
print
print '=============================='
print
class FileIndex:
SIGNATURE = b'BOB1'
CACHE_ENTRY_FMT = '=QQLqLQ20sH'
CACHE_ENTRY_SIZE = struct.calcsize(CACHE_ENTRY_FMT)
class Stat:
def __init__(self):
self.name = b""
self.ctime = 0
self.mtime = 0
self.dev = 0
self.ino = 0
self.mode = 0
self.size = 0
self.digest = b''
def __init__(self, cachePath):
self.__cachePath = cachePath
self.__cacheDir = os.path.dirname(cachePath)
def open(self):
self.__inPos = 0
self.__inPosOld = 0
self.__outFile = None
try:
if os.path.exists(self.__cachePath):
self.__inFile = open(self.__cachePath, "rb")
sig = self.__inFile.read(4)
if sig == DirHasher.FileIndex.SIGNATURE:
self.__mismatch = False
self.__inPos = self.__inPosOld = 4
else:
logging.getLogger(__name__).info(
"Wrong signature at '%s': %s", self.__cachePath,
sig)
self.__inFile.close()
self.__inFile = None
self.__mismatch = True
else:
self.__inFile = None
self.__mismatch = True
except OSError as e:
raise BuildError("Error opening hash cache: " + str(e))
self.__current = DirHasher.FileIndex.Stat()
def close(self):
try:
if self.__inFile:
self.__inFile.close()
if self.__outFile:
self.__outFile.close()
os.replace(self.__outFile.name, self.__cachePath)
except OSError as e:
raise BuildError("Error closing hash cache: " + str(e))
def __readEntry(self):
if not self.__inFile: return False
raw = self.__inFile.read(DirHasher.FileIndex.CACHE_ENTRY_SIZE)
if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: return False
e = self.__current
(e.ctime, e.mtime, e.dev, e.ino, e.mode, e.size, e.digest,
nameLen) = struct.unpack(DirHasher.FileIndex.CACHE_ENTRY_FMT, raw)
e.name = self.__inFile.read(nameLen)
self.__inPosOld = self.__inPos
self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen
return True
def __writeEntry(self, name, st, digest):
if not self.__outFile:
self.__outFile = NamedTemporaryFile(mode="wb",
dir=self.__cacheDir,
delete=False)
if self.__inFile:
pos = self.__inFile.tell()
self.__inFile.seek(0)
self.__outFile.write(self.__inFile.read(self.__inPosOld))
self.__inFile.seek(pos)
else:
self.__outFile.write(DirHasher.FileIndex.SIGNATURE)
self.__outFile.write(
struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT,
float2ns(st.st_ctime), float2ns(st.st_mtime),
st.st_dev, st.st_ino, st.st_mode, st.st_size,
digest, len(name)))
self.__outFile.write(name)
def __match(self, name, st):
while self.__current.name < name:
if not self.__readEntry(): break
e = self.__current
res = ((e.name == name) and (e.ctime == float2ns(st.st_ctime))
and (e.mtime == float2ns(st.st_mtime))
and (e.dev == st.st_dev) and (e.ino == st.st_ino)
and (e.mode == st.st_mode) and (e.size == st.st_size))
#if not res: print("Mismatch", e.name, name)
return res
def check(self, prefix, name, st, process):
if self.__match(name, st):
digest = self.__current.digest
else:
digest = process(os.path.join(prefix, name))
self.__mismatch = True
if self.__mismatch:
self.__writeEntry(name, st, digest)
return digest
"""
def sample_run(input_file_path):
with cp2k.ForceEnvironment(input_file_path, "/dev/null") as fenv:
print("potential energy: {:e}".format(fenv.potential_energy))
print("calculating energy..")
fenv.calc_energy()
print(".. done!")
print("potential energy: {:e}".format(fenv.potential_energy))
print("positions:")
pos = fenv.positions
print(pos)
print("zeroify positions..")
fenv.positions = np.zeros(pos.shape)
print("positions:")
print(fenv.positions)
if __name__ == "__main__":
cp2k.init()
tempfile = NamedTemporaryFile(mode="w+")
tempfile.write(TEST_FILE_CONTENT)
tempfile.flush()
sample_run(tempfile.name)
cp2k.finalize()
def _write_local_data_files(self, cursor):
"""
Takes a cursor, and writes results to a local file.
:return: A dictionary where keys are filenames to be used as object
names in GCS, and values are file handles to local files that
contain the data for the GCS objects.
"""
schema = list(
map(lambda schema_tuple: schema_tuple[0], cursor.description))
col_type_dict = self._get_col_type_dict()
file_no = 0
tmp_file_handle = NamedTemporaryFile(delete=True)
if self.export_format == 'csv':
file_mime_type = 'text/csv'
elif self.export_format == 'parquet':
file_mime_type = 'application/octet-stream'
else:
file_mime_type = 'application/json'
files_to_upload = [{
'file_name': self.filename.format(file_no),
'file_handle': tmp_file_handle,
'file_mime_type': file_mime_type,
}]
self.log.info("Current file count: %d", len(files_to_upload))
if self.export_format == 'csv':
csv_writer = self._configure_csv_file(tmp_file_handle, schema)
if self.export_format == 'parquet':
parquet_schema = self._convert_parquet_schema(cursor)
parquet_writer = self._configure_parquet_file(
tmp_file_handle, parquet_schema)
for row in cursor:
# Convert datetime objects to utc seconds, and decimals to floats.
# Convert binary type object to string encoded with base64.
row = self.convert_types(schema, col_type_dict, row)
if self.export_format == 'csv':
if self.null_marker is not None:
row = [
value if value is not None else self.null_marker
for value in row
]
csv_writer.writerow(row)
elif self.export_format == 'parquet':
if self.null_marker is not None:
row = [
value if value is not None else self.null_marker
for value in row
]
row_pydic = {col: [value] for col, value in zip(schema, row)}
tbl = pa.Table.from_pydict(row_pydic, parquet_schema)
parquet_writer.write_table(tbl)
else:
row_dict = dict(zip(schema, row))
tmp_file_handle.write(
json.dumps(row_dict, sort_keys=True,
ensure_ascii=False).encode("utf-8"))
# Append newline to make dumps BigQuery compatible.
tmp_file_handle.write(b'\n')
# Stop if the file exceeds the file size limit.
if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
file_no += 1
tmp_file_handle = NamedTemporaryFile(delete=True)
files_to_upload.append({
'file_name':
self.filename.format(file_no),
'file_handle':
tmp_file_handle,
'file_mime_type':
file_mime_type,
})
self.log.info("Current file count: %d", len(files_to_upload))
if self.export_format == 'csv':
csv_writer = self._configure_csv_file(
tmp_file_handle, schema)
if self.export_format == 'parquet':
parquet_writer = self._configure_parquet_file(
tmp_file_handle, parquet_schema)
return files_to_upload
class K8SCopyToPod(K8SCopy):
"""
Copy files/directory from local filesystem into remote Pod
"""
def __init__(self, module, client):
super(K8SCopyToPod, self).__init__(module, client)
self.files_to_copy = list()
def run_from_pod(self, command):
response = stream(self.api_instance.connect_get_namespaced_pod_exec,
self.name,
self.namespace,
command=command,
stderr=True,
stdin=False,
stdout=True,
tty=False,
_preload_content=False,
**self.container_arg)
errors = []
while response.is_open():
response.update(timeout=1)
if response.peek_stderr():
errors.append(response.read_stderr())
response.close()
err = response.read_channel(ERROR_CHANNEL)
err = yaml.safe_load(err)
response.close()
if err['status'] != 'Success':
self.module.fail_json(
msg="Failed to run {0} on Pod.".format(command), errors=errors)
def is_remote_path_dir(self):
pod_command = ['test', '-d', self.remote_path]
response = stream(self.api_instance.connect_get_namespaced_pod_exec,
self.name,
self.namespace,
command=pod_command,
stdout=True,
stderr=True,
stdin=False,
tty=False,
_preload_content=False,
**self.container_arg)
while response.is_open():
response.update(timeout=1)
err = response.read_channel(ERROR_CHANNEL)
err = yaml.safe_load(err)
response.close()
if err['status'] == 'Success':
return True
return False
def close_temp_file(self):
if self.named_temp_file:
self.named_temp_file.close()
def run(self):
try:
# remove trailing slash from destination path
dest_file = self.remote_path.rstrip("/")
src_file = self.local_path
self.named_temp_file = None
if self.content:
self.named_temp_file = NamedTemporaryFile(mode="w")
self.named_temp_file.write(self.content)
self.named_temp_file.flush()
src_file = self.named_temp_file.name
else:
if not os.path.exists(self.local_path):
self.module.fail_json(
msg="{0} does not exist in local filesystem".format(
self.local_path))
if not os.access(self.local_path, os.R_OK):
self.module.fail_json(
msg="{0} not readable".format(self.local_path))
if self.is_remote_path_dir():
if self.content:
self.module.fail_json(
msg=
"When content is specified, remote path should not be an existing directory"
)
else:
dest_file = os.path.join(dest_file,
os.path.basename(src_file))
if self.no_preserve:
tar_command = [
'tar', '--no-same-permissions', '--no-same-owner', '-xmf',
'-'
]
else:
tar_command = ['tar', '-xmf', '-']
if dest_file.startswith("/"):
tar_command.extend(['-C', '/'])
response = stream(
self.api_instance.connect_get_namespaced_pod_exec,
self.name,
self.namespace,
command=tar_command,
stderr=True,
stdin=True,
stdout=True,
tty=False,
_preload_content=False,
**self.container_arg)
with TemporaryFile() as tar_buffer:
with tarfile.open(fileobj=tar_buffer, mode='w') as tar:
tar.add(src_file, dest_file)
tar_buffer.seek(0)
commands = []
# push command in chunk mode
size = 1024 * 1024
while True:
data = tar_buffer.read(size)
if not data:
break
commands.append(data)
stderr, stdout = [], []
while response.is_open():
if response.peek_stdout():
stdout.append(response.read_stdout().rstrip("\n"))
if response.peek_stderr():
stderr.append(response.read_stderr().rstrip("\n"))
if commands:
cmd = commands.pop(0)
response.write_stdin(cmd)
else:
break
response.close()
if stderr:
self.close_temp_file()
self.module.fail_json(
command=tar_command,
msg=
"Failed to copy local file/directory into Pod due to: {0}"
.format(''.join(stderr)))
self.close_temp_file()
if self.content:
self.module.exit_json(
changed=True,
result="Content successfully copied into {0} on remote Pod"
.format(self.remote_path))
self.module.exit_json(
changed=True,
result="{0} successfully copied into remote Pod into {1}".
format(self.local_path, self.remote_path))
except Exception as e:
self.module.fail_json(
msg="Failed to copy local file/directory into Pod due to: {0}".
format(to_native(e)))
def run_long_running(params, is_test=False):
"""
Start the long running server
:param params: Demisto params
:param is_test: Indicates whether it's test-module run or regular run
:return: None
"""
certificate: str = params.get('certificate', '')
private_key: str = params.get('key', '')
certificate_path = str()
private_key_path = str()
try:
port = get_params_port(params)
ssl_args = dict()
if (certificate and not private_key) or (private_key
and not certificate):
raise DemistoException(
'If using HTTPS connection, both certificate and private key should be provided.'
)
if certificate and private_key:
certificate_file = NamedTemporaryFile(delete=False)
certificate_path = certificate_file.name
certificate_file.write(bytes(certificate, 'utf-8'))
certificate_file.close()
private_key_file = NamedTemporaryFile(delete=False)
private_key_path = private_key_file.name
private_key_file.write(bytes(private_key, 'utf-8'))
private_key_file.close()
context = SSLContext(PROTOCOL_TLSv1_2)
context.load_cert_chain(certificate_path, private_key_path)
ssl_args['ssl_context'] = context
demisto.debug('Starting HTTPS Server')
else:
demisto.debug('Starting HTTP Server')
server = WSGIServer(('0.0.0.0', port),
APP,
**ssl_args,
log=DEMISTO_LOGGER)
if is_test:
server_process = Process(target=server.serve_forever)
server_process.start()
time.sleep(5)
server_process.terminate()
else:
server.serve_forever()
except SSLError as e:
ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}'
demisto.error(ssl_err_message)
raise ValueError(ssl_err_message)
except Exception as e:
demisto.error(f'An error occurred in long running loop: {str(e)}')
raise ValueError(str(e))
finally:
if certificate_path:
os.unlink(certificate_path)
if private_key_path:
os.unlink(private_key_path)
class FileSourceTest(unittest.TestCase):
def setUp(self):
dirlist = ["plugins/datagraph"]
self.loader = PluginsLoader(Application)
self.loader.load(dirlist)
self._dataFile = NamedTemporaryFile(mode='w',
delete=False,
encoding='utf8')
def tearDown(self):
self.loader.clear()
self._dataFile.close()
os.remove(self._dataFile.name)
self._dataFile = None
def testFileSource_empty_01(self):
from datagraph.datasources import FileSource
data = ''''''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0, items)
def testFileSource_empty_02(self):
from datagraph.datasources import FileSource
data = '''
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
def testFileSource_single_01(self):
from datagraph.datasources import FileSource
data = '''123'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['123'])
def testFileSource_single_02(self):
from datagraph.datasources import FileSource
data = '''123 '''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['123'])
def testFileSource_single_03(self):
from datagraph.datasources import FileSource
data = ''' 123 '''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['123'])
def testFileSource_single_04(self):
from datagraph.datasources import FileSource
data = '''123
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['123'])
def testFileSource_single_05(self):
from datagraph.datasources import FileSource
data = '''123
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['123'])
def testFileSource_one_col_01(self):
from datagraph.datasources import FileSource
data = '''123
234
456'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 3)
self.assertEqual(items[0], ['123'])
self.assertEqual(items[1], ['234'])
self.assertEqual(items[2], ['456'])
def testFileSource_one_col_02(self):
from datagraph.datasources import FileSource
data = '''123
234
456
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 3)
self.assertEqual(items[0], ['123'])
self.assertEqual(items[1], ['234'])
self.assertEqual(items[2], ['456'])
def testFileSource_one_col_03(self):
from datagraph.datasources import FileSource
data = '''123
234
456
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 3)
self.assertEqual(items[0], ['123'])
self.assertEqual(items[1], ['234'])
self.assertEqual(items[2], ['456'])
def testFileSource_one_col_04(self):
from datagraph.datasources import FileSource
data = '''
123
234
4560
1000
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 4)
self.assertEqual(items[0], ['123'])
self.assertEqual(items[1], ['234'])
self.assertEqual(items[2], ['4560'])
self.assertEqual(items[3], ['1000'])
def testFileSource_col_01(self):
from datagraph.datasources import FileSource
data = '''123 456 789
234 100 111
456 101 99
-10\t55 66
20 30 40 '''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 5)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
self.assertEqual(items[2], ['456', '101', '99'])
self.assertEqual(items[3], ['-10', '55', '66'])
self.assertEqual(items[4], ['20', '30', '40'])
def testFileSource_col_02(self):
from datagraph.datasources import FileSource
data = '''
123 456 789
234 100 111
456 101 99
-10\t55 66
20 30 40
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 5)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
self.assertEqual(items[2], ['456', '101', '99'])
self.assertEqual(items[3], ['-10', '55', '66'])
self.assertEqual(items[4], ['20', '30', '40'])
def testFileSource_col_03(self):
from datagraph.datasources import FileSource
data = '''123 456 789
234 100 111
456 101
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 2)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
def testFileSource_col_04(self):
from datagraph.datasources import FileSource
data = '''123 456 789
234 100 111
456 101 99 78
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 2)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
def testFileSource_skiprows_01(self):
from datagraph.datasources import FileSource
data = '''
123 456 789
234 100 111
456 101 99
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 3)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
def testFileSource_skiprows_02(self):
from datagraph.datasources import FileSource
data = '''Абырвалг
----
123 456 789
234 100 111
456 101 99
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name, skiprows=2)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 3)
self.assertEqual(items[0], ['123', '456', '789'])
self.assertEqual(items[1], ['234', '100', '111'])
def testFileSource_skiprows_03(self):
from datagraph.datasources import FileSource
data = '''Абырвалг Абыр
----
123 456 789
234 100 111
456 101 99
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name, skiprows=0)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 1)
self.assertEqual(items[0], ['Абырвалг', 'Абыр'])
def testFileSource_skiprows_04(self):
from datagraph.datasources import FileSource
data = '''Абырвалг Абыр
----
123 456 789
234 100 111
456 101 99
'''
self._dataFile.write(data)
self._dataFile.flush()
source = FileSource(self._dataFile.name, skiprows=5)
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
def testFileSource_invalid_01(self):
from datagraph.datasources import FileSource
source = FileSource('invalid_fname.txt')
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
def testFileSource_invalid_02(self):
from datagraph.datasources import FileSource
source = FileSource('testdata/samplefiles/image.png')
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
def testFileSource_invalid_03(self):
from datagraph.datasources import FileSource
source = FileSource('testdata/samplefiles/invalid.exe')
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
def testFileSource_invalid_04(self):
from datagraph.datasources import FileSource
source = FileSource('testdata/samplefiles/text_1251.txt')
items = list(source.getRowsIterator())
self.assertEqual(len(items), 0)
