def test_read_not_bytes(self):
source = NamedTemporaryFile(mode="w+")
source.write("a bunch of text")
source.seek(0)
with self.assertRaises(ValueError):
self.tree._read(source)
def get_tempfile(self, **kwargs):
kwargs.setdefault('suffix', '.vrt')
tempfile = NamedTemporaryFile(**kwargs)
tempfile.write(self.content)
tempfile.flush()
tempfile.seek(0)
return tempfile
def _generate_training_files(self):
"""Returns a tuple of file objects suitable for passing to the
RdpTrainer application controller.
"""
tmp_dir = get_qiime_temp_dir()
training_set = RdpTrainingSet()
reference_seqs_file = open(self.Params['reference_sequences_fp'], 'U')
id_to_taxonomy_file = open(self.Params['id_to_taxonomy_fp'], 'U')
for seq_id, seq in MinimalFastaParser(reference_seqs_file):
training_set.add_sequence(seq_id, seq)
for line in id_to_taxonomy_file:
seq_id, lineage_str = map(strip, line.split('\t'))
training_set.add_lineage(seq_id, lineage_str)
training_set.dereplicate_taxa()
rdp_taxonomy_file = NamedTemporaryFile(
prefix='RdpTaxonAssigner_taxonomy_', suffix='.txt', dir=tmp_dir)
rdp_taxonomy_file.write(training_set.get_rdp_taxonomy())
rdp_taxonomy_file.seek(0)
rdp_training_seqs_file = NamedTemporaryFile(
prefix='RdpTaxonAssigner_training_seqs_', suffix='.fasta',
dir=tmp_dir)
for rdp_id, seq in training_set.get_training_seqs():
rdp_training_seqs_file.write('>%s\n%s\n' % (rdp_id, seq))
rdp_training_seqs_file.seek(0)
self._training_set = training_set
return rdp_taxonomy_file, rdp_training_seqs_file
def generate_shscript(args):
""" Generate sh script input file"""
header = """\
#!/bin/bash
#PBS -q {0}
#PBS -l nodes={1}:ppn={2}:native
#PBS -l walltime={3}
#PBS -N {4}
#PBS -o {4}.out
#PBS -e {4}.err
#PBS -A {5}
#PBS -M {6}
#PBS -m abe
#PBS -V
""".format(args.queue, args.nodes, args.ppn, args.walltime,
args.run_name, args.account, args.email)
header = dedent(header)
print header + args.cmd
f = NamedTemporaryFile(delete=False)
f.write(header + args.cmd)
f.seek(0)
# for i in f:
# print i.strip()
return f.name
def create_temp_file(self, edid_binary):
edid_file = NamedTemporaryFile(delete=False)
edid_file.write(edid_binary)
edid_file.flush()
edid_file.seek(0)
return edid_file
class TestTrio(object):
"""Test class for testing how the individual class behave"""
def setup_class(self):
"""Setup a standard trio."""
trio_lines = ['#Standard trio\n',
'#FamilyID\tSampleID\tFather\tMother\tSex\tPhenotype\n',
'healthyParentsAffectedSon\tproband\tfather\tmother\t1\t2\n',
'healthyParentsAffectedSon\tmother\t0\t0\t2\t1\n',
'healthyParentsAffectedSon\tfather\t0\t0\t1\t1\n'
]
self.trio_file = NamedTemporaryFile(mode='w+t', delete=False, suffix='.vcf')
self.trio_file.writelines(trio_lines)
self.trio_file.seek(0)
self.trio_file.close()
def test_standard_trio(self):
"""Test if the file is parsed in a correct way."""
family_parser = parser.FamilyParser(open(self.trio_file.name, 'r'))
assert family_parser.header == [
'family_id',
'sample_id',
'father_id',
'mother_id',
'sex',
'phenotype'
]
assert 'healthyParentsAffectedSon' in family_parser.families
assert set(['proband', 'mother', 'father']) == set(family_parser.families['healthyParentsAffectedSon'].individuals.keys())
assert set(['proband', 'mother', 'father']) == set(family_parser.families['healthyParentsAffectedSon'].trios[0])
def reg_code():
img, code =generate_code_image((80,30),5)
session["code"] = code
tp = NamedTemporaryFile()
img.save(tp.name,format="png")
tp.seek(0)
return send_file(tp.name,mimetype='image/png')
def setup_vcf_file():
"""
Print some variants to a vcf file and return the filename
"""
vcf_lines = [
'##fileformat=VCFv4.1\n',
'##INFO=<ID=MQ,Number=1,Type=Float,Description="RMS Mapping Quality">\n',
'##contig=<ID=1,length=249250621,assembly=b37>\n',
'##reference=file:///humgen/gsa-hpprojects/GATK/bundle'\
'/current/b37/human_g1k_v37.fasta\n',
'#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tfather\tmother\tproband\n',
'1\t11900\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/1:60\t0/1:60\t1/1:60\n',
'1\t879585\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/1:60\t0/0:60\t0/1:60\n',
'1\t879586\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/1:60\t0/1:60\n',
'1\t947378\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/0:60\t0/1:60\n',
'1\t973348\t.\tG\tA\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/0:60\t0/1:60\n',
'3\t879585\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/1:60\t0/0:60\t0/1:60\n',
'3\t879586\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/1:60\t0/1:60\n',
'3\t947378\t.\tA\tT\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/0:60\t0/1:60\n',
'3\t973348\t.\tG\tA\t100\tPASS\tMQ=1\tGT:GQ\t0/0:60\t0/0:60\t0/1:60\n'
]
vcf_file = NamedTemporaryFile(mode='w+t', delete=False, suffix='.vcf')
vcf_file.writelines(vcf_lines)
vcf_file.seek(0)
vcf_file.close()
return vcf_file.name
def write_temp_file(data):
# create a temp file for use as a config file. This should get cleaned
# up magically at the end of the run.
fid = NamedTemporaryFile(mode='w+b', suffix='.tmp')
fid.write(data)
fid.seek(0)
return fid
def test_dump_load(self):
from tempfile import NamedTemporaryFile
handle = NamedTemporaryFile()
self.discodb.dump(handle)
handle.seek(0)
discodb = DiscoDB.load(handle)
self.assertEquals(discodb.dumps(), self.discodb.dumps())
def get_subtitle(url, path):
in_data = urllib2.urlopen(url)
temp_file = NamedTemporaryFile()
temp_file.write(in_data.read())
in_data.close()
temp_file.seek(0)
if is_zipfile(temp_file.name):
zip_file = ZipFile(temp_file)
for name in zip_file.namelist():
# don't unzip stub __MACOSX folders
if '.srt' in name and '__MACOSX' not in name:
logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)]))
zip_file.extract(name, os.path.dirname(path))
zip_file.close()
elif is_rarfile(temp_file.name):
rar_path = path + '.rar'
logger.info('Saving rared subtitle as %s' % rar_path)
with open(rar_path, 'w') as out_file:
out_file.write(temp_file.read())
try:
import subprocess
#extract all .srt in the rared file
ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path])
if ret_code == 0:
logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path))
os.remove(rar_path)
except OSError:
logger.info('Unpacking rared subtitle failed.'
'Please, install unrar to automate this step.')
temp_file.close()
def _ufopen(self, _url, _filename ):
'''Open url and save in tempfile'''
import urllib
__t = NamedTemporaryFile(prefix='',suffix= _filename )
__t.write( urllib.urlopen(_url).read())
__t.seek(0)
return __t.name, __t
def run_as(user, command, *args, **kwargs):
"""
Run a command as a particular user, using ``/etc/environment`` and optionally
capturing and returning the output.
Raises subprocess.CalledProcessError if command fails.
:param str user: Username to run command as
:param str command: Command to run
:param list args: Additional args to pass to command
:param dict env: Additional env variables (will be merged with ``/etc/environment``)
:param bool capture_output: Capture and return output (default: False)
:param str input: Stdin for command
"""
parts = [command] + list(args)
quoted = ' '.join("'%s'" % p for p in parts)
env = read_etc_env()
if 'env' in kwargs:
env.update(kwargs['env'])
run = check_output if kwargs.get('capture_output') else check_call
try:
stdin = None
if 'input' in kwargs:
stdin = NamedTemporaryFile()
stdin.write(kwargs['input'])
stdin.seek(0)
return run(['su', user, '-c', quoted], env=env, stdin=stdin)
finally:
if stdin:
stdin.close() # this also removes tempfile
def write_temp_file(text, encoding='utf-8', delete=True):
tmp = NamedTemporaryFile(delete=delete)
if type(text) == unicode:
text = text.encode(encoding, 'ignore')
tmp.write(text)
tmp.seek(0)
return File(tmp)
def __init__(self, srv, lookup, userdb, spconf, url, return_to, verification_endpoint="verify", cache=None, bindings=None):
"""
Constructor for the class.
:param srv: Usually none, but otherwise the oic server.
:param return_to: The URL to return to after a successful
authentication.
"""
self.userdb = userdb
if cache is None:
self.cache_outstanding_queries = {}
else:
self.cache_outstanding_queries = cache
UserAuthnMethod.__init__(self, srv)
self.return_to = return_to
self.idp_query_param = "IdpQuery"
if bindings:
self.bindings = bindings
else:
self.bindings = [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST,
BINDING_HTTP_ARTIFACT]
self.verification_endpoint = verification_endpoint
#Configurations for the SP handler. (pyOpSamlProxy.client.sp.conf)
self.sp_conf = importlib.import_module(spconf)
#self.sp_conf.BASE = self.sp_conf.BASE % url
ntf = NamedTemporaryFile(suffix="pyoidc.py", delete=True)
ntf.write("CONFIG = " + str(self.sp_conf.CONFIG).replace("%s", url))
ntf.seek(0)
self.sp = Saml2Client(config_file="%s" % ntf.name)
mte = lookup.get_template("unauthorized.mako")
argv = {
"message": "You are not authorized!",
}
self.not_authorized = mte.render(**argv)
def _create_zip(self, assignment, zip_rootdir_name):
tempfile = NamedTemporaryFile()
zip_file = zipfile.ZipFile(tempfile, 'w');
for group in self._get_queryset(assignment):
candidates = self._get_candidates_as_string(group)
for deadline in group.deadlines.all():
for delivery in deadline.deliveries.all():
for filemeta in delivery.filemetas.all():
file_content = filemeta.deliverystore.read_open(filemeta)
filenametpl = '{zip_rootdir_name}/{groupname}/deadline-{deadline}/delivery-{delivery_number}/{filename}'
if candidates:
groupname = '{candidates}_group-{groupid}'.format(
candidates=candidates,
groupid=group.id
)
else:
groupname = 'group-{groupid}'.format(groupid=group.id)
filename = filenametpl.format(zip_rootdir_name=zip_rootdir_name,
groupname=groupname,
deadline=deadline.deadline.strftime(self.DEADLINE_FORMAT),
delivery_number="%.3d" % delivery.number,
filename = filemeta.filename.encode(ZIPFILE_FILENAME_CHARSET))
zip_file.writestr(filename, file_content.read())
zip_file.close()
tempfile.seek(0)
return tempfile
def main():
"""Implement the "rose config-dump" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode")
opts = opt_parser.parse_args()[0]
verbosity = opts.verbosity - opts.quietness
report = Reporter(verbosity)
fs_util = FileSystemUtil(report)
if opts.conf_dir:
fs_util.chdir(opts.conf_dir)
file_names = []
if opts.files:
file_names = opts.files
else:
for dirpath, _, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, "rose-*.conf"):
path = os.path.join(dirpath, filename)[2:] # remove leading ./
file_names.append(path)
for file_name in file_names:
handle = NamedTemporaryFile()
node = ConfigLoader()(file_name)
if (not opts.no_pretty_mode and
os.path.basename(file_name) != META_CONFIG_NAME):
pretty_format_config(node, ignore_error=True)
ConfigDumper()(node, handle)
handle.seek(0)
if not filecmp.cmp(handle.name, file_name, shallow=False):
report(ConfigDumpEvent(file_name))
ConfigDumper()(node, file_name)
def make_image_file(dimensions=(320, 240), extension=".jpeg", force_size=None, orientation=None):
"""
Yields a named temporary file created with the specified image type and
options.
Note the default dimensions are unequal (not a square) ensuring that center-square
cropping logic will be exercised during tests.
The temporary file will be closed and deleted automatically upon exiting
the `with` block.
"""
image = Image.new('RGB', dimensions, "green")
image_file = NamedTemporaryFile(suffix=extension)
try:
if orientation and orientation in xrange(1, 9):
exif_bytes = piexif.dump({'0th': {piexif.ImageIFD.Orientation: orientation}})
image.save(image_file, exif=exif_bytes)
else:
image.save(image_file)
if force_size is not None:
image_file.seek(0, os.SEEK_END)
bytes_to_pad = force_size - image_file.tell()
# write in hunks of 256 bytes
hunk, byte_ = bytearray([0] * 256), bytearray([0])
num_hunks, remainder = divmod(bytes_to_pad, 256)
for _ in xrange(num_hunks):
image_file.write(hunk)
for _ in xrange(remainder):
image_file.write(byte_)
image_file.flush()
image_file.seek(0)
yield image_file
finally:
image_file.close()
def module(minion, name, *args, **kwargs):
"""
Execute an arbitrary salt module on dev minion.
"""
script = '/usr/local/sbin/minion-' + minion
if not os.access(script, os.X_OK):
raise exc.CommandNotFoundError("Unknown minion " + minion)
args = [script, name] + list(args)
for k, v in kwargs.items():
if k.startswith('__'):
continue
args.append('%s=%s' % (k, v))
logger.debug("Calling %r", args)
stderr = NamedTemporaryFile(
prefix=script + '-' + name + '-', suffix='.stderr')
child = subprocess.Popen(
args, stdout=stderr, stderr=stderr)
child.wait()
stderr.seek(0)
out = stderr.read()
stderr.close()
if child.returncode != 0:
raise exc.CommandExecutionError(out)
return out
def export(request, thegallery):
"""Export a gallery to a zip file and send it to the user.
"""
# Check if the gallery is valid
gallery = get_object_or_404(Gallery, pk=thegallery)
# gather up the photos into a new directory
tmpdir = mkdtemp()
for photo in gallery.photo_set.all():
shutil.copy(photo.get_image_filename(),
tmpdir)
files = [ os.path.join(tmpdir, ff) for ff in os.listdir(tmpdir) ]
outfile = NamedTemporaryFile()
zf = zipfile.ZipFile(outfile, "w",
compression=zipfile.ZIP_DEFLATED)
for filename in files:
zf.write(filename, arcname=os.path.basename(filename))
zf.close()
outfile.flush()
outfile.seek(0)
shutil.rmtree(tmpdir)
response = HttpResponse(outfile)
response['Content-Type'] = "application/zip"
response['Content-Length'] = str(os.stat(outfile.name)[stat.ST_SIZE])
response['Content-Disposition'] = "attachment; filename=photos.zip"
return response
def export_csv_for_model(model, dataset):
model_cls = getattr(udata_models, model.capitalize(), None)
if not model_cls:
log.error('Unknow model %s' % model)
return
queryset = get_queryset(model_cls)
adapter = csv.get_adapter(model_cls)
if not adapter:
log.error('No adapter found for %s' % model)
return
adapter = adapter(queryset)
log.info('Exporting CSV for %s...' % model)
csvfile = NamedTemporaryFile(delete=False)
try:
# write adapter results into a tmp file
writer = csv.get_writer(csvfile)
writer.writerow(adapter.header())
for row in adapter.rows():
writer.writerow(row)
csvfile.seek(0)
# make a resource from this tmp file
created, resource = store_resource(csvfile, model, dataset)
# add it to the dataset
if created:
dataset.add_resource(resource)
dataset.last_modified = datetime.now()
dataset.save()
finally:
os.unlink(csvfile.name)
def xls_export(request, username, id_string):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u"Not shared."))
query = request.GET.get("query")
force_xlsx = request.GET.get("xlsx") == "true"
xls_df_builder = XLSDataFrameBuilder(username, id_string, query)
excel_defs = {
"xls": {"suffix": ".xls", "mime_type": "vnd.ms-excel"},
"xlsx": {"suffix": ".xlsx", "mime_type": "vnd.openxmlformats"}, # TODO: check xlsx mime type
}
ext = "xls" if not force_xlsx else "xlsx"
if xls_df_builder.exceeds_xls_limits:
ext = "xlsx"
try:
temp_file = NamedTemporaryFile(suffix=excel_defs[ext]["suffix"])
xls_df_builder.export_to(temp_file.name)
if request.GET.get("raw"):
id_string = None
response = response_with_mimetype_and_name(excel_defs[ext]["mime_type"], id_string, extension=ext)
response.write(temp_file.read())
temp_file.seek(0, os.SEEK_END)
response["Content-Length"] = temp_file.tell()
temp_file.close()
return response
except NoRecordsFoundError:
return HttpResponse(_("No records found to export"))
def export(self, out_f=None, format='mp3'):
out_f = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
data = NamedTemporaryFile(mode="wb", delete=False)
wave_data = wave.open(data)
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
wave_data.setnframes(self.frame_count())
wave_data.writeframesraw(self._data)
wave_data.close()
output = NamedTemporaryFile(mode="w+")
# read stdin / write stdout
subprocess.call(['ffmpeg',
'-y', # always overwrite existing files
"-f", "wav", "-i", data.name, # input options (filename last)
"-f", format, output.name, # output options (filename last)
],
# make ffmpeg shut up
stderr=open(os.devnull))
output.seek(0)
out_f.write(output.read())
data.unlink(data.name)
out_f.seek(0)
return out_f
def main():
"""Implement the "rose config-dump" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode")
opts, args = opt_parser.parse_args()
verbosity = opts.verbosity - opts.quietness
report = Reporter(verbosity)
fs_util = FileSystemUtil(report)
if opts.conf_dir:
fs_util.chdir(opts.conf_dir)
file_names = []
if opts.files:
file_names = opts.files
else:
for dirpath, dirnames, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, "rose-*.conf"):
p = os.path.join(dirpath, filename)[2:] # remove leading ./
file_names.append(p)
for file_name in file_names:
t = NamedTemporaryFile()
node = ConfigLoader()(file_name)
if not opts.no_pretty_mode:
pretty_format_config(node)
ConfigDumper()(node, t)
t.seek(0)
if not filecmp.cmp(t.name, file_name, shallow=False):
report(ConfigDumpEvent(file_name))
ConfigDumper()(node, file_name)
def test_merge_sam():
'It merges two sams'
reference = NamedTemporaryFile(suffix='.sam')
reference.write('''>SGN-U572743
atatata
>SGN-U576692
gcgc''')
sam1 = NamedTemporaryFile(suffix='.sam')
sam1.write('''@SQ SN:SGN-U576692 LN:1714
@SQ SN:SGN-U572743 LN:833
@RG ID:g1 LB:g1 SM:g1
@RG ID:g2 LB:g2 SM:g2
SGN-E221403 0 SGN-U576692 1416 207 168M * 0 0 AGCCTGATAAAGGTCTGCCTACGTGTTTTAAGTGGAATCCGTTTCCCCATGTCCAAACCTTCTAAATAGTTTTTTGTGTTAGTTCTTGTATGCCACATACAAAAATTAACAAACTCTTTTGCCACATATGTTCCAGCACGTCAAAGCAACATGTATTTGAGCTACTTT 558<///035EB@;550300094>>FBF>>88>BBB200>@FFMMMJJ@@755225889>0..14444::FMF@@764444448@;;84444<//,4,.,<<QFBB;::/,,,.69FBB>9:2/.409;@@>88.7,//55;BDK@11,,093777777884241<:7 AS:i:160 XS:i:0 XF:i:3 XE:i:4 XN:i:0 RG:Z:g2
SGN-E221664 0 SGN-U572743 317 226 254M24S * 0 0 GGATGATCTTAGAGCTGCCATTCAAAAGATGTTAGACACTCCTGGGCCATACTTGTTGGATGTGATTGTACCTCATCAGGAGCATGTTCTACCGATGATTCCCAGTGGCGGTGCTTTCAAAAATGTGATTACGGAGGGTGATGGGAGACGTTCCTATTGACTTTGAGAAGCTACATAACTAGTTCAAGGCATTGTATTATCTAAAATAAACTTAATATTTATGTTTACTTAAAAGTTTTTCATTGTGTGAAGGAAAAAAAAAAAAAAAAAAAAAAAAA 999@7<22-2***-,206433>:?9<,,,66:>00066=??EEAAA?B200002<<@@@=DB99777864:..0::@833099???<@488>></...<:B<<.,,8881288@BBDDBD885@@;;9:/9.,,,99B99233885558=?DKKKDDAA??DKBB=440/0<8?DEDFBB??6@152@@FBMFIIDDDDDDKKKOK@@@@DD:N688BBDDDBBBKKDEDDBN977?<9<111:<??==BKMPKKBB==99>QQYYYYYYYYYYYYQQ AS:i:250 XS:i:0 XF:i:0 XE:i:7 XN:i:0 RG:Z:g1
''')
sam1.flush()
sam2 = NamedTemporaryFile(suffix='.sam')
sam2.write('''@SQ SN:SGN-U576692 LN:1714
@SQ SN:SGN-U572743 LN:833
@RG ID:g1 LB:g1 SM:g1
@RG ID:g3 LB:g3 SM:g3
SGN-E200000 0 SGN-U572743 317 226 254M24S * 0 0 GGATGATCTTAGAGKTGCCATTCAAAAGATGTTAGACACTCCTGGGCCATACTTGTTGGATGTGATTGTACCTCATCAGGAGCATGTTCTACCGATGATTCCCAGTGGCGGTGCTTTCAAAAATGTGATTACGGAGGGTGATGGGAGACGTTCCTATTGACTTTGAGAAGCTACATAACTAGTTCAAGGCATTGTATTATCTAAAATAAACTTAATATTTATGTTTACTTAAAAGTTTTTCATTGTGTGAAGGAAAAAAAAAAAAAAAAAAAAAAAAA 999@7<22-2***-,206433>:?9<,,,66:>00066=??EEAAA?B200002<<@@@=DB99777864:..0::@833099???<@488>></...<:B<<.,,8881288@BBDDBD885@@;;9:/9.,,,99B99233885558=?DKKKDDAA??DKBB=440/0<8?DEDFBB??6@152@@FBMFIIDDDDDDKKKOK@@@@DD:N688BBDDDBBBKKDEDDBN977?<9<111:<??==BKMPKKBB==99>QQYYYYYYYYYYYYQQ AS:i:250 XS:i:0 XF:i:0 XE:i:7 XN:i:0 RG:Z:g1
SGN-E40000 0 SGN-U576692 1416 207 168M * 0 0 AGCCTGATAAAGGTCTGCCTACGTGTTTTAAGTGGAATCCGTTTCCCCATGTCCAAACCTTCTAAATAGTTTTTTGTGTTAGTTCTTGTATGCCACATACAAAAATTAACAAACTCTTTTGCCACATATGTTCCAGCACGTCAAAGCAACATGTATTTGAGCTACTTT 558<///035EB@;550300094>>FBF>>88>BBB200>@FFMMMJJ@@755225889>0..14444::FMF@@764444448@;;84444<//,4,.,<<QFBB;::/,,,.69FBB>9:2/.409;@@>88.7,//55;BDK@11,,093777777884241<:7 AS:i:160 XS:i:0 XF:i:3 XE:i:4 XN:i:0 RG:Z:g3
''')
sam2.flush()
sam3 = NamedTemporaryFile(suffix='.sam')
merge_sam(infiles=[sam1, sam2], outfile=sam3, reference=reference)
sam3.seek(0)
sam3_content = sam3.read()
assert 'SN:SGN-U572743' in sam3_content
assert 'SGN-E200000' in sam3_content
assert 'SGN-E221664' in sam3_content
def get(self, request, deliveryid):
delivery = get_object_or_404(Delivery, id=deliveryid)
assignment_group = delivery.deadline.assignment_group
if not (assignment_group.is_candidate(request.user) \
or assignment_group.is_examiner(request.user) \
or request.user.is_superuser \
or assignment_group.parentnode.is_admin(request.user)):
return HttpResponseForbidden("Forbidden")
dirname = '{}-{}-delivery{}'.format(
assignment_group.parentnode.get_path(),
assignment_group.get_candidates(separator='_'),
delivery.number)
zip_file_name = u'{}.zip'.format(dirname)
tempfile = NamedTemporaryFile()
zip_file = zipfile.ZipFile(tempfile, 'w');
for filemeta in delivery.filemetas.all():
file_content = filemeta.deliverystore.read_open(filemeta)
zip_file.write(file_content.name, posixpath.join(dirname, filemeta.filename))
zip_file.close()
tempfile.seek(0)
response = HttpResponse(FileWrapperWithExplicitClose(tempfile),
content_type="application/zip")
response['Content-Disposition'] = "attachment; filename=%s" % \
zip_file_name.encode("ascii", 'replace')
response['Content-Length'] = stat(tempfile.name).st_size
return response
def decrypt(self, enctext):
""" Decrypting an encrypted text by the use of a private key.
:param enctext: The encrypted text as a string
:return: The decrypted text
"""
if self.log:
self.log.info("input len: %d" % len(enctext))
_, fil = make_temp("%s" % enctext, decode=False)
ntf = NamedTemporaryFile(delete=not DEBUG)
com_list = [self.xmlsec, "--decrypt",
"--privkey-pem", self.key_file,
"--output", ntf.name,
"--id-attr:%s" % ID_ATTR, ENC_KEY_CLASS,
fil]
if self.debug:
self.log.debug("Decrypt command: %s" % " ".join(com_list))
pof = Popen(com_list, stderr=PIPE, stdout=PIPE)
p_out = pof.stdout.read()
p_err = pof.stderr.read()
if self.debug:
self.log.debug("Decrypt result (out): %s" % (p_out,))
self.log.debug("Decrypt result (err): %s" % (p_err,))
ntf.seek(0)
return ntf.read()
def handle_say(self, engine, message, cmd, args, lang='sonid26', voice='Alyona'):
phrase = args.strip().encode('utf-8')
if not len(phrase):
raise Exception(
'Введіть, що сказати!'
'Приклад: /report Мегас поц'
)
tk = gtts_token.Token().calculate_token(phrase)
query = urllib.urlencode(dict(
MyLanguages=lang,
MySelectedVoice=voice,
MyTextForTTS=phrase,
t=1,
SendToVaaS=''
# q=phrase,
# ie='UTF-8',
# tl=lang,
# total=1,
# textlen=len(phrase),
# tk=tk,
# client='t',
# ttsspeed=0.5
))
url = 'http://www.acapela-group.com/demo-tts/DemoHTML5Form_V2.php'
request = urllib2.Request(url, query)
request.add_header('Referer', 'http://www.acapela-group.com/')
# request.add_header('Accept-Encoding', 'identity;q=1, *;q=0')
request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36')
# request.add_header('Range', 'bytes=0-')
response = urllib2.urlopen(request)
html = response.read()
url = re.findall('http.*\.mp3', html)
if not len(url):
raise Exception('API послав нас нах :/')
url = url[0]
response = urllib2.urlopen(url.strip())
data = response.read()
af = NamedTemporaryFile(delete=False, suffix='.mp3')
af.write(data)
af.seek(0)
af = open(af.name, 'rb')
try:
engine.telegram.sendAudio(
chat_id=message.chat_id,
audio=af,
title=phrase
)
os.unlink(af.name)
return True
except:
os.unlink(af.name)
raise
return True
def src_file():
src = NamedTemporaryFile()
src.write(data)
src.flush()
src.seek(0, SEEK_SET)
return src
def test_basic_behaviour(self):
'It tests that we can get names using a FileNamingSchema'
fhand = NamedTemporaryFile()
engine = sqlalchemy.create_engine('sqlite:///:memory:')
create_naming_database(engine)
add_project_to_naming_database(engine, name='my_project', code='my',
description='a test project')
naming = DbNamingSchema(engine, project='my_project',
feature_kind='EST')
naming = FileNamingSchema(fhand, naming)
assert naming.get_uniquename(name='hola') == 'myES000001'
assert naming.get_uniquename(name='hola') == 'myES000001'
assert naming.get_uniquename(name='caracol') == 'myES000002'
naming.commit()
fhand.seek(0)
naming = FileNamingSchema(fhand)
naming.kind = 'EST'
assert naming.get_uniquename(name='hola') == 'myES000001'
assert naming.get_uniquename(name='caracol') == 'myES000002'
try:
assert naming.get_uniquename(name='pascual')
self.fail()
#pylint: disable-msg=W0704
except ValueError:
pass
def FFT(job):
rawFileHash = job.get('hash')
segmentHash = job.data['segment-hash']
# WAV: The number of channel is contained in the shape of data (len, numChannels)
sampleFrequency, rdata = read(
filepath.GetSegmentFromHash(rawFileHash, segmentHash))
# Convert to complex for the FFT
data = rdata.astype(np.complex_)
fftSampleFrenquency = 80
fftSampleLength = math.ceil(0.25 * sampleFrequency)
hamming = np.reshape(np.repeat(np.hamming(fftSampleLength), data.shape[1]),
(fftSampleLength, 2))
# Number of FFT to compute
total = math.floor(
(len(data) - fftSampleLength) * fftSampleFrenquency / sampleFrequency)
# We don't have the file hash yet
filepath.EnsureExists('/data/temp/file')
file = NamedTemporaryFile(mode='w+b', dir='/data/temp', delete=False)
# File header, [number of FFT Samples, FFT sample length, original file sample frequency, FFT sample frequency]
file.write(np.zeros(4, dtype=np.int32).tobytes())
print('[fft] Data shape:', data.shape)
# Start the job
job.start(total)
# Debug purpose only
allData = np.array([], dtype=np.float_)
index = 0
while math.ceil(index * sampleFrequency /
fftSampleFrenquency) + fftSampleLength <= len(data):
s = math.ceil(index * sampleFrequency / fftSampleFrenquency)
e = math.ceil(
index * sampleFrequency / fftSampleFrenquency) + fftSampleLength
fftData = spfft.fft(data[s:e] * hamming)
fftPowerData = np.sum(np.absolute(fftData, dtype=np.float_), axis=1)
fftPowerData = 1.0 / fftSampleLength * fftPowerData**2
# TODO: We could ignore the later half of the data (negative frequencies) as it's symmetric for FFT of real signals
file.write(fftPowerData.tobytes())
# print(file.tell(), np.amax(fftPowerData))
# Debug purpose only
allData = np.append(allData, fftPowerData)
index = index + 1
job.update(index)
# Debug purpose only
plt.imshow(np.reshape(allData, (fftSampleLength, -1)), aspect='auto')
filepath.EnsureExists(
filepath.GetFFTFromHash(rawFileHash, segmentHash) + '.svg')
plt.savefig(filepath.GetFFTFromHash(rawFileHash, segmentHash) + '.svg')
file.seek(0)
file.write(
np.array(
[index, fftSampleLength, sampleFrequency, fftSampleFrenquency],
dtype=np.int32).tobytes())
file.close()
print('[fft] Processing complete')
print('[fft] Finalizing...')
# Rename the temporary file
fftHash = filepath.Hash(file.name)
filename = filepath.GetFFTFromHash(rawFileHash, fftHash)
filepath.EnsureExists(filename)
os.rename(file.name, filename)
job.data['fft-hash'] = fftHash
job.complete()
redis.open().publish('JOB-UPDATE-FFT', job.serialize())
print('[fft] [SUCCESS] Job', rawFileHash, '-', segmentHash)
print('[fft] [SUCCESS] Job result', rawFileHash, '-', fftHash)
def export(self,
out_f=None,
format='mp3',
codec=None,
bitrate=None,
parameters=None,
tags=None,
id3v2_version='4',
cover=None):
"""
Export an AudioSegment to a file with given options
out_f (string):
Path to destination audio file. Also accepts os.PathLike objects on
python >= 3.6
format (string)
Format for destination audio file.
('mp3', 'wav', 'raw', 'ogg' or other ffmpeg/avconv supported files)
codec (string)
Codec used to encode the destination file.
bitrate (string)
Bitrate used when encoding destination file. (64, 92, 128, 256, 312k...)
Each codec accepts different bitrate arguments so take a look at the
ffmpeg documentation for details (bitrate usually shown as -b, -ba or
-a:b).
parameters (list of strings)
Aditional ffmpeg/avconv parameters
tags (dict)
Set metadata information to destination files
usually used as tags. ({title='Song Title', artist='Song Artist'})
id3v2_version (string)
Set ID3v2 version for tags. (default: '4')
cover (file)
Set cover for audio file from image file. (png or jpg)
"""
id3v2_allowed_versions = ['3', '4']
if format == "raw" and (codec is not None or parameters is not None):
raise AttributeError(
'Can not invoke ffmpeg when export format is "raw"; '
'specify an ffmpeg raw format like format="s16le" instead '
'or call export(format="raw") with no codec or parameters')
out_f, _ = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
if format == "raw":
out_f.write(self._data)
out_f.seek(0)
return out_f
# wav with no ffmpeg parameters can just be written directly to out_f
easy_wav = format == "wav" and codec is None and parameters is None
if easy_wav:
data = out_f
else:
data = NamedTemporaryFile(mode="wb", delete=False)
pcm_for_wav = self._data
if self.sample_width == 1:
# convert to unsigned integers for wav
pcm_for_wav = audioop.bias(self._data, 1, 128)
wave_data = wave.open(data, 'wb')
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
# For some reason packing the wave header struct with
# a float in python 2 doesn't throw an exception
wave_data.setnframes(int(self.frame_count()))
wave_data.writeframesraw(pcm_for_wav)
wave_data.close()
# for easy wav files, we're done (wav data is written directly to out_f)
if easy_wav:
out_f.seek(0)
return out_f
output = NamedTemporaryFile(mode="w+b", delete=False)
# build converter command to export
conversion_command = [
self.converter,
'-y', # always overwrite existing files
"-f",
"wav",
"-i",
data.name, # input options (filename last)
]
if codec is None:
codec = self.DEFAULT_CODECS.get(format, None)
if cover is not None:
if cover.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tif',
'.tiff')) and format == "mp3":
conversion_command.extend(
["-i", cover, "-map", "0", "-map", "1", "-c:v", "mjpeg"])
else:
raise AttributeError(
"Currently cover images are only supported by MP3 files. The allowed image formats are: .tif, .jpg, .bmp, .jpeg and .png."
)
if codec is not None:
# force audio encoder
conversion_command.extend(["-acodec", codec])
if bitrate is not None:
conversion_command.extend(["-b:a", bitrate])
if parameters is not None:
# extend arguments with arbitrary set
conversion_command.extend(parameters)
if tags is not None:
if not isinstance(tags, dict):
raise InvalidTag("Tags must be a dictionary.")
else:
# Extend converter command with tags
# print(tags)
for key, value in tags.items():
conversion_command.extend(
['-metadata', '{0}={1}'.format(key, value)])
if format == 'mp3':
# set id3v2 tag version
if id3v2_version not in id3v2_allowed_versions:
raise InvalidID3TagVersion(
"id3v2_version not allowed, allowed versions: %s" %
id3v2_allowed_versions)
conversion_command.extend(
["-id3v2_version", id3v2_version])
if sys.platform == 'darwin' and codec == 'mp3':
conversion_command.extend(["-write_xing", "0"])
conversion_command.extend([
"-f",
format,
output.name, # output options (filename last)
])
log_conversion(conversion_command)
# read stdin / write stdout
with open(os.devnull, 'rb') as devnull:
p = subprocess.Popen(conversion_command,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p_out, p_err = p.communicate()
log_subprocess_output(p_out)
log_subprocess_output(p_err)
if p.returncode != 0:
raise CouldntEncodeError(
"Encoding failed. ffmpeg/avlib returned error code: {0}\n\nCommand:{1}\n\nOutput from ffmpeg/avlib:\n\n{2}"
.format(p.returncode, conversion_command,
p_err.decode(errors='ignore')))
output.seek(0)
out_f.write(output.read())
data.close()
output.close()
os.unlink(data.name)
os.unlink(output.name)
out_f.seek(0)
return out_f
import requests
import shutil
import os.path
from glob import glob
from tempfile import NamedTemporaryFile
DATA_URL = 'http://stash.compjour.org/data/sunlight-twitter-opener.zip'
DATA_DIR = 'data-hold'
tzip = NamedTemporaryFile(suffix='.zip')
print("Downloading", DATA_URL)
r = requests.get(DATA_URL)
tzip.write(r.content)
tzip.seek(0)
print("Unzipping to", DATA_DIR)
shutil.unpack_archive(tzip.name, DATA_DIR)
tzip.close()
csvname = os.path.join(DATA_DIR, 'sunlight_legislators.csv')
print(csvname, 'has', os.path.getsize(csvname), 'bytes')
print("Tweet files:", len(glob('data-hold/tweets/*.json')))
print("Profile files:", len(glob('data-hold/profiles/*.json')))
def write_inputs(
self,
X: "npt.ArrayLike",
y: "npt.ArrayLike",
groups: "Optional[npt.ArrayLike]" = None,
covariates: "Optional[npt.ArrayLike]" = None,
) -> "Iterator":
from tempfile import NamedTemporaryFile
X_file = NamedTemporaryFile(mode="w+")
y_file = NamedTemporaryFile(mode="w+")
if (covariates is not None) or (groups is not None):
cov_file: "Optional[IO[str]]" = NamedTemporaryFile(mode="w+")
else:
cov_file = None
try:
# self._check_is_singular(np.asarray(X))
X_ = self.genos_to_bimbam(X)
X_.to_csv(X_file, index=False, header=False, na_rep="NA")
X_file.seek(0)
y_ = np.asarray(y)
if len(y_.shape) == 1:
y_ = np.expand_dims(y_, -1)
pd.DataFrame(y_).to_csv(
y_file,
index=False,
header=False,
na_rep="NA"
)
y_file.seek(0)
cov: "List[np.ndarray]" = []
if groups is not None:
cov.append(self.prep_groups(groups))
if covariates is not None:
cov.append(np.asarray(covariates))
if len(cov) > 0:
"""
We add a column of ones to include an intercept term.
"""
assert cov_file is not None
cov_ = np.concatenate(
[np.ones((y_.shape[0], 1)), *cov],
axis=1
)
pd.DataFrame(cov_).to_csv(
cov_file,
index=False,
header=False,
na_rep="NA"
)
cov_file.seek(0)
del X_
del y_
del cov
yield X_file, y_file, cov_file
finally:
X_file.close()
y_file.close()
if cov_file is not None:
cov_file.close()
return
class Warc(object):
MAIN_URL = "WARC-X-Main-URL"
_main_url = None
_file_name = None
_warc_file_read = None
_warc_file_write = None
_temporary = None
_read_only = False
def __init__(self, file_name, temporary=False, read_only=False, **kwargs):
self._main_url = kwargs.get("main_url")
self._file_name = file_name
self._temporary = temporary
self._read_only = read_only if not self._temporary else False
if self._temporary:
if not self._main_url:
raise ValueError("Missing required argument: main_url")
self._warc_file_read = NamedTemporaryFile("rb")
self._warc_file_write = open(self._warc_file_read.name, "wb")
self._init_file()
else:
if self._read_only:
self._warc_file_read = open(file_name, "rb")
else:
self._warc_file_read = open(file_name, "rb")
self._warc_file_write = open(file_name, "ab")
def find_record(self, url):
self._warc_file_read.seek(0)
wrs = WarcRecord.open_archive(file_handle=self._warc_file_read, \
gzip="record")
for (offset, record, errors) in wrs.read_records(limit=None):
if record and (record.type == WarcRecord.RESPONSE) \
and (record.content[0] == ResponseMessage.CONTENT_TYPE) \
and (record.url == url):
return record
return None
def write_record(self, record):
if self._read_only:
raise RuntimeError("WARC opened for read-only access")
self._warc_file_write.seek(0, os.SEEK_END)
record.write_to(self._warc_file_write, gzip=True)
self._warc_file_write.flush()
def make_permanent(self):
if not self._temporary:
raise RuntimeError("This WARC is not temporary")
warc_file = open(self._file_name, "wb")
self._warc_file_read.seek(0)
# copy temp file to it's permanent location
shutil.copyfileobj(self._warc_file_read, warc_file)
warc_file.flush()
self._warc_file_read = open(self._file_name, "rb")
self._warc_file_write = warc_file
self._temporary = False
@property
def main_url(self):
return self._main_url
@property
def temporary(self):
return self._temporary
@property
def read_only(self):
return self._read_only
def _init_file(self):
warcinfo_headers = [
(WarcRecord.TYPE, WarcRecord.WARCINFO),
(WarcRecord.ID, WarcRecord.random_warc_uuid()),
(WarcRecord.DATE, warc.warc_datetime_str(datetime.utcnow())),
(WarcRecord.FILENAME, os.path.basename(self._file_name)),
(Warc.MAIN_URL, self._main_url),
]
warcinfo_fields = "\r\n".join([
"software: bardo",
"format: WARC File Format 1.0",
"conformsTo: " + CONFORMS_TO,
"robots: unknown",
])
warcinfo_content = ("application/warc-fields", warcinfo_fields)
warcinfo_record = WarcRecord(headers=warcinfo_headers, \
content=warcinfo_content)
self.write_record(warcinfo_record)
def _load_warc_info(self):
self._warc_file_read.seek(0)
wrs = WarcRecord.open_archive(file_handle=self._warc_file_read, \
gzip="record")
temp = wrs.read_records(limit=1)
if not temp or (temp[0].type != WarcRecord.WARCINFO):
raise ValueError("WARC info not found")
return temp[0]
class LazyZipOverHTTP:
"""File-like object mapped to a ZIP file over HTTP.
This uses HTTP range requests to lazily fetch the file's content,
which is supposed to be fed to ZipFile. If such requests are not
supported by the server, raise HTTPRangeRequestUnsupported
during initialization.
"""
def __init__(self, url, session, chunk_size=CONTENT_CHUNK_SIZE):
# type: (str, PipSession, int) -> None
head = session.head(url, headers=HEADERS)
raise_for_status(head)
assert head.status_code == 200
self._session, self._url, self._chunk_size = session, url, chunk_size
self._length = int(head.headers['Content-Length'])
self._file = NamedTemporaryFile()
self.truncate(self._length)
self._left = [] # type: List[int]
self._right = [] # type: List[int]
if 'bytes' not in head.headers.get('Accept-Ranges', 'none'):
raise HTTPRangeRequestUnsupported('range request is not supported')
self._check_zip()
@property
def mode(self):
# type: () -> str
"""Opening mode, which is always rb."""
return 'rb'
@property
def name(self):
# type: () -> str
"""Path to the underlying file."""
return self._file.name
def seekable(self):
# type: () -> bool
"""Return whether random access is supported, which is True."""
return True
def close(self):
# type: () -> None
"""Close the file."""
self._file.close()
@property
def closed(self):
# type: () -> bool
"""Whether the file is closed."""
return self._file.closed
def read(self, size=-1):
# type: (int) -> bytes
"""Read up to size bytes from the object and return them.
As a convenience, if size is unspecified or -1,
all bytes until EOF are returned. Fewer than
size bytes may be returned if EOF is reached.
"""
download_size = max(size, self._chunk_size)
start, length = self.tell(), self._length
stop = length if size < 0 else min(start + download_size, length)
start = max(0, stop - download_size)
self._download(start, stop - 1)
return self._file.read(size)
def readable(self):
# type: () -> bool
"""Return whether the file is readable, which is True."""
return True
def seek(self, offset, whence=0):
# type: (int, int) -> int
"""Change stream position and return the new absolute position.
Seek to offset relative position indicated by whence:
* 0: Start of stream (the default). pos should be >= 0;
* 1: Current position - pos may be negative;
* 2: End of stream - pos usually negative.
"""
return self._file.seek(offset, whence)
def tell(self):
# type: () -> int
"""Return the current possition."""
return self._file.tell()
def truncate(self, size=None):
# type: (Optional[int]) -> int
"""Resize the stream to the given size in bytes.
If size is unspecified resize to the current position.
The current stream position isn't changed.
Return the new file size.
"""
return self._file.truncate(size)
def writable(self):
# type: () -> bool
"""Return False."""
return False
def __enter__(self):
# type: () -> LazyZipOverHTTP
self._file.__enter__()
return self
def __exit__(self, *exc):
# type: (*Any) -> Optional[bool]
return self._file.__exit__(*exc)
@contextmanager
def _stay(self):
# type: ()-> Iterator[None]
"""Return a context manager keeping the position.
At the end of the block, seek back to original position.
"""
pos = self.tell()
try:
yield
finally:
self.seek(pos)
def _check_zip(self):
# type: () -> None
"""Check and download until the file is a valid ZIP."""
end = self._length - 1
for start in reversed(range(0, end, self._chunk_size)):
self._download(start, end)
with self._stay():
try:
# For read-only ZIP files, ZipFile only needs
# methods read, seek, seekable and tell.
ZipFile(self) # type: ignore
except BadZipfile:
pass
else:
break
def _stream_response(self, start, end, base_headers=HEADERS):
# type: (int, int, Dict[str, str]) -> Response
"""Return HTTP response to a range request from start to end."""
headers = base_headers.copy()
headers['Range'] = f'bytes={start}-{end}'
# TODO: Get range requests to be correctly cached
headers['Cache-Control'] = 'no-cache'
return self._session.get(self._url, headers=headers, stream=True)
def _merge(self, start, end, left, right):
# type: (int, int, int, int) -> Iterator[Tuple[int, int]]
"""Return an iterator of intervals to be fetched.
Args:
start (int): Start of needed interval
end (int): End of needed interval
left (int): Index of first overlapping downloaded data
right (int): Index after last overlapping downloaded data
"""
lslice, rslice = self._left[left:right], self._right[left:right]
i = start = min([start] + lslice[:1])
end = max([end] + rslice[-1:])
for j, k in zip(lslice, rslice):
if j > i:
yield i, j - 1
i = k + 1
if i <= end:
yield i, end
self._left[left:right], self._right[left:right] = [start], [end]
def _download(self, start, end):
# type: (int, int) -> None
"""Download bytes from start to end inclusively."""
with self._stay():
left = bisect_left(self._right, start)
right = bisect_right(self._left, end)
for start, end in self._merge(start, end, left, right):
response = self._stream_response(start, end)
response.raise_for_status()
self.seek(start)
for chunk in response_chunks(response, self._chunk_size):
self._file.write(chunk)
class S3KeyWritableFileObject(RawIOBase):
def __init__(
self, object_summery, *,
path,
mode='w',
buffering=DEFAULT_BUFFER_SIZE,
encoding=None,
errors=None,
newline=None):
super().__init__()
self.object_summery = object_summery
self.path = path
self.mode = mode
self.buffering = buffering
self.encoding = encoding
self.errors = errors
self.newline = newline
self._cache = NamedTemporaryFile(
mode=self.mode + '+' if 'b' in self.mode else 'b' + self.mode + '+',
buffering=self.buffering,
encoding=self.encoding,
newline=self.newline)
self._string_parser = partial(_string_parser, mode=self.mode, encoding=self.encoding)
def __getattr__(self, item):
try:
return getattr(self._cache, item)
except AttributeError:
return super().__getattribute__(item)
def writable_check(method):
@wraps(method)
def wrapper(self, *args, **kwargs):
if not self.writable():
raise UnsupportedOperation('not writable')
return method(self, *args, **kwargs)
return wrapper
def writable(self, *args, **kwargs):
return 'w' in self.mode
@writable_check
def write(self, text):
self._cache.write(self._string_parser(text))
self._cache.seek(0)
_s3_accessor.boto3_method_with_parameters(
self.object_summery.put,
kwargs={'Body': self._cache}
)
def writelines(self, lines):
if not lines:
return
if isinstance(lines[0], bytes):
joined = b"".join(lines)
else:
joined = "".join(lines)
self.write(joined)
def readable(self):
return False
def read(self, *args, **kwargs):
raise UnsupportedOperation('not readable')
def readlines(self, *args, **kwargs):
raise UnsupportedOperation('not readable')
def main(input_file, output_file, debug):
if input_file == '-':
if debug:
raise BaseException("cannot run --debug with stdin input")
input_file = sys.stdin
else:
input_file = open(input_file)
# store / filter input
tmpf = NamedTemporaryFile()
# created digest
output = NamedTemporaryFile()
# mailbox needs filename :( that's breaking duck typing!
# with below From hack I could probably split msgs myself altogether
for line in input_file:
# mutt pipe-to doesn't do mbox format :( this will break soon :)
if line.startswith('Delivered-To: '):
tmpf.write('From [email protected] Thu Aug 15 16:24:28 2019\n')
tmpf.write(line)
tmpf.flush()
tmpf.seek(0)
mbox = mailbox.mbox(tmpf.name)
# transform headers to dict, lowercase and merge multiline headers, decode
# quoted-printable bodies
mbox_usable = []
for msg in mbox:
mbox_usable.append(MyMail())
mbox_usable[-1].headers = normalize_headers(msg._headers)
# Get the only part or the text/plain one if there are multiple
payload = msg.get_payload()
if isinstance(payload, type([])):
for submsg in payload:
subpart_headers = normalize_headers(submsg._headers)
if subpart_headers['content-type'].startswith('text/plain'):
# add headers from multipart subpart
mbox_usable[-1].headers.update(subpart_headers)
mbox_usable[-1].body = maybe_decode(
mbox_usable[-1].headers['content-transfer-encoding'],
submsg.get_payload())
break
else:
print "warning: message doesn't have text/plain part"
else:
mbox_usable[-1].body = maybe_decode(
mbox_usable[-1].headers['content-transfer-encoding'], payload)
mbox_usable.sort(key=lambda x: parsedate(x.headers['date']))
if debug:
code.interact(local=locals())
first = True
for msg in mbox_usable:
if first is True:
print >> output, '>____________________________________________________________________<'
print >> output, 'Date: ', msg.headers.get('date')
print >> output, 'Subject: ', msg.headers.get('subject')
print >> output
print >> output, msg.body
first = False
else:
print >> output, '>____________________________________________________________________<'
print >> output, 'Date: ', msg.headers.get('date')
print >> output, filter_stuff(msg.body)
output.flush()
tmpf.close()
os.system("vim -c 'set ft=mail' -c 'set wrap' '%s'" % output.name)
except Exception, ex:
print 'Error reading file %s.' % outfile
print 'Error: ', ex
print 'Deleting and recaching file.'
os.remove(outfile)
warnings.resetwarnings()
if not os.path.exists(outfile):
if not self.quiet: print 'Caching ft1 file. Saving to %s' % outfile
if isinstance(self.sa.pixeldata.ft1files, collections.Iterable):
temp = NamedTemporaryFile(delete=True)
temp.write('\n'.join(self.sa.pixeldata.ft1files))
temp.seek(0)
infile = '@%s' % temp.name
else:
infile = self.sa.pixeldata.ft1files
import GtApp
GtApp.GtApp("gtselect",
'dataSubselector').run(infile=infile,
outfile=outfile,
ra=self.roi_dir.ra(),
dec=self.roi_dir.dec(),
rad=self.sa.maxROI,
tmin=0,
tmax=0,
emin=emin,
emax=emax,
def create(self, request):
"""
Does not work in Swagger!
Create a new Property from a building file.
---
consumes:
- multipart/form-data
parameters:
- name: organization_id
type: integer
required: true
- name: cycle_id
type: integer
required: true
- name: file_type
type: string
enum: ["Unknown", "BuildingSync", "HPXML"]
required: true
- name: file
description: In-memory file object
required: true
type: file
"""
if len(request.FILES) == 0:
return JsonResponse({
'success':
False,
'message':
'Must pass file in as a Multipart/Form post'
})
the_file = request.data['file']
file_type = BuildingFile.str_to_file_type(
request.data.get('file_type', 'Unknown'))
organization_id = request.data['organization_id']
cycle = request.data.get('cycle_id', None)
if not cycle:
return JsonResponse({
'success': False,
'message': 'Cycle ID is not defined'
})
else:
cycle = Cycle.objects.get(pk=cycle)
# figure out if file is xml or zip
the_filename = the_file._get_name()
tmp_filename, file_extension = os.path.splitext(the_filename)
# initialize
p_status = True
property_state = True
messages = {'errors': [], 'warnings': []}
if file_extension == '.zip':
# ZIP FILE, extract and process files one by one
# print("This file is a ZIP")
with zipfile.ZipFile(the_file, "r", zipfile.ZIP_STORED) as openzip:
filelist = openzip.infolist()
for f in filelist:
# print("FILE: {}".format(f.filename))
# process xml files
if '.xml' in f.filename and '__MACOSX' not in f.filename:
# print("PROCESSING file: {}".format(f.filename))
data_file = NamedTemporaryFile()
data_file.write(openzip.read(f))
data_file.seek(0)
size = os.path.getsize(data_file.name)
content_type = 'text/xml'
# print("DATAFILE:")
# print(data_file)
a_file = InMemoryUploadedFile(data_file,
'data_file',
f.filename,
content_type,
size,
charset=None)
building_file = BuildingFile.objects.create(
file=a_file,
filename=f.filename,
file_type=file_type,
)
p_status_tmp, property_state_tmp, property_view, messages_tmp = building_file.process(
organization_id, cycle)
# print('messages_tmp: ')
# print(messages_tmp)
# append errors to overall messages
for i in messages_tmp['errors']:
messages['errors'].append(f.filename + ": " + i)
for i in messages_tmp['warnings']:
messages['warnings'].append(f.filename + ": " + i)
if not p_status_tmp:
# capture error
p_status = p_status_tmp
else:
# capture a real property_state (not None)
property_state = property_state_tmp
else:
# just an XML
building_file = BuildingFile.objects.create(
file=the_file,
filename=the_file.name,
file_type=file_type,
)
p_status, property_state, property_view, messages = building_file.process(
organization_id, cycle)
if p_status and property_state:
if len(messages['warnings']) > 0:
return JsonResponse({
'success': True,
'status': 'success',
'message': {
'warnings': messages['warnings']
},
'data': {
'property_view':
PropertyViewAsStateSerializer(property_view).data,
# 'property_state': PropertyStateWritableSerializer(property_state).data,
},
})
else:
return JsonResponse({
'success': True,
'status': 'success',
'message': {
'warnings': []
},
'data': {
'property_view':
PropertyViewAsStateSerializer(property_view).data,
# 'property_state': PropertyStateWritableSerializer(property_state).data,
},
})
else:
return JsonResponse(
{
'success': False,
'status': 'error',
'message': messages
},
status=status.HTTP_400_BAD_REQUEST)
class TestMothurClassifySeqs(TestCase):
def setUp(self):
self.ref_file = NamedTemporaryFile()
self.ref_file.write(mothur_ref_seqs)
self.ref_file.seek(0)
self.tax_file = NamedTemporaryFile()
self.tax_file.write(mothur_taxonomy)
self.tax_file.seek(0)
def test_app(self):
app = MothurClassifySeqs({
'reference': self.ref_file.name,
'taxonomy': self.tax_file.name,
})
res = app(mothur_seqs)
assignments = res['assignments'].read()
self.assertEqual(assignments, mothur_assignments)
summary = res['summary'].read()
self.assertEqual(summary, mothur_summary)
def test_format_function_arguments(self):
app = MothurClassifySeqs({
'reference': '/home/myuser/ref-seqs.fasta',
'taxonomy': '/home/MyUser/data/tax.txt',
'cutoff': 80,
})
obs_args = app._format_function_arguments(
['reference', 'taxonomy', 'cutoff', 'iters'])
exp_args = ("reference=/home/myuser/ref\\-seqs.fasta, "
"taxonomy=/home/MyUser/data/tax.txt, cutoff=80")
self.assertEqual(obs_args, exp_args)
def test_compile_mothur_script(self):
app = MothurClassifySeqs({
'reference': '/home/myuser/ref-seqs.fasta',
'taxonomy': '/home/MyUser/data/tax.txt',
'cutoff': 80,
})
app._input_filename = "/my/input.fasta"
exp_script = ("#classify.seqs(fasta=/my/input.fasta, "
"reference=/home/myuser/ref\-seqs.fasta, "
"taxonomy=/home/MyUser/data/tax.txt, "
"cutoff=80)")
self.assertEqual(app._compile_mothur_script(), exp_script)
def test_mothur_classify_file(self):
query_file = StringIO(mothur_seqs)
res = mothur_classify_file(query_file, self.ref_file.name,
self.tax_file.name)
exp_res = {
'A': ([
'k__Bacteria', 'p__Firmicutes', 'c__Clostridia',
'o__Clostridale', 'f__Eubacteriaceae', 'g__Eubacterium',
's__Eubacteriumfoedans'
], 1.0),
'Very': (['k__Bacteria', 'p__Bacteriodetes'], 1.0),
'01': (['k__Bacteria', 'p__Firmicutes'], 1.0),
}
self.assertEqual(res, exp_res)
def test_unclassifiable_sequence(self):
query_file = StringIO(
">MostlyTs\nTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"
"TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTATTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"
"TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT\n"
)
res = mothur_classify_file(query_file, self.ref_file.name,
self.tax_file.name)
exp_res = {
'MostlyTs': (['Unknown'], 0.0),
}
self.assertEqual(res, exp_res)
def export(self,
out_f=None,
format='mp3',
codec=None,
bitrate=None,
parameters=None,
tags=None,
id3v2_version='4'):
"""
Export an AudioSegment to a file with given options
out_f (string):
Path to destination audio file
format (string)
Format for destination audio file.
('mp3', 'wav', 'raw', 'ogg' or other ffmpeg/avconv supported files)
codec (string)
Codec used to encoding for the destination.
bitrate (string)
Bitrate used when encoding destination file. (64, 92, 128, 256, 312k...)
Each codec accepts different bitrate arguments so take a look at the
ffmpeg documentation for details (bitrate usually shown as -b, -ba or
-a:b).
parameters (string)
Aditional ffmpeg/avconv parameters
tags (dict)
Set metadata information to destination files
usually used as tags. ({title='Song Title', artist='Song Artist'})
id3v2_version (string)
Set ID3v2 version for tags. (default: '4')
"""
id3v2_allowed_versions = ['3', '4']
out_f = _fd_or_path_or_tempfile(out_f, 'wb+')
out_f.seek(0)
if format == "raw":
out_f.write(self._data)
out_f.seek(0)
return out_f
# for wav output we can just write the data directly to out_f
if format == "wav":
data = out_f
else:
data = NamedTemporaryFile(mode="wb", delete=False)
wave_data = wave.open(data, 'wb')
wave_data.setnchannels(self.channels)
wave_data.setsampwidth(self.sample_width)
wave_data.setframerate(self.frame_rate)
# For some reason packing the wave header struct with
# a float in python 2 doesn't throw an exception
wave_data.setnframes(int(self.frame_count()))
wave_data.writeframesraw(self._data)
wave_data.close()
# for wav files, we're done (wav data is written directly to out_f)
if format == 'wav':
return out_f
output = NamedTemporaryFile(mode="w+b", delete=False)
# build converter command to export
conversion_command = [
self.converter,
'-y', # always overwrite existing files
"-f",
"wav",
"-i",
data.name, # input options (filename last)
]
if codec is None:
codec = self.DEFAULT_CODECS.get(format, None)
if codec is not None:
# force audio encoder
conversion_command.extend(["-acodec", codec])
if bitrate is not None:
conversion_command.extend(["-b:a", bitrate])
if parameters is not None:
# extend arguments with arbitrary set
conversion_command.extend(parameters)
if tags is not None:
if not isinstance(tags, dict):
raise InvalidTag("Tags must be a dictionary.")
else:
# Extend converter command with tags
# print(tags)
for key, value in tags.items():
conversion_command.extend(
['-metadata', '{0}={1}'.format(key, value)])
if format == 'mp3':
# set id3v2 tag version
if id3v2_version not in id3v2_allowed_versions:
raise InvalidID3TagVersion(
"id3v2_version not allowed, allowed versions: %s" %
id3v2_allowed_versions)
conversion_command.extend(
["-id3v2_version", id3v2_version])
conversion_command.extend([
"-f",
format,
output.name, # output options (filename last)
])
log_conversion(conversion_command)
# read stdin / write stdout
p = subprocess.Popen(conversion_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p_out, p_err = p.communicate()
if p.returncode != 0:
raise CouldntEncodeError(
"Encoding failed. ffmpeg/avlib returned error code: {0}\n\nOutput from ffmpeg/avlib:\n\n{1}"
.format(p.returncode, p_err))
output.seek(0)
out_f.write(output.read())
data.close()
output.close()
os.unlink(data.name)
os.unlink(output.name)
out_f.seek(0)
return out_f
def o4_pyforce(debug, no_revision, args: list, quiet=False):
"""
Encapsulates Pyforce, does book keeping to ensure that all files
that should be operated on are in fact dealt with by p4. Handles
retry and strips out asks for files that are caseful mismatches on
the current file system (macOS).
"""
from tempfile import NamedTemporaryFile
from collections import defaultdict
class LogAndAbort(Exception):
'Dumps debug information on errors.'
o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args)
tmpf = NamedTemporaryFile(dir='.o4')
fstats = []
for line in sys.stdin.read().splitlines():
if line.startswith('#o4pass'):
print(line)
continue
f = fstat_split(line)
if f and caseful_accurate(f[F_PATH]):
fstats.append(f)
elif f:
print(
f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully",
"mismatching a local file.",
file=sys.stderr)
retries = 3
head = _depot_path().replace('/...', '')
while fstats:
if no_revision:
p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats]
else:
p4paths = [
f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats
]
tmpf.seek(0)
tmpf.truncate()
not_yet = []
pargs = []
xargs = []
# This is a really bad idea, files are output to stdout before the actual
# sync happens, causing checksum tests to start too early:
# if len(p4paths) > 30 and 'sync' in args:
# xargs.append('--parallel=threads=5')
if sum(len(s) for s in p4paths) > 30000:
pargs.append('-x')
pargs.append(tmpf.name)
for f in p4paths:
tmpf.write(f.encode('utf8'))
tmpf.write(b'\n')
tmpf.flush()
else:
xargs.extend(p4paths)
try:
# TODO: Verbose
#print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join(
# repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else ''))
errs = []
repeats = defaultdict(list)
infos = []
for res in Pyforce(*pargs, *args, *xargs):
if debug:
err_print("*** DEBUG: Received", repr(res))
# FIXME: Delete this if-statement:
if res.get('code', '') == 'info':
infos.append(res)
if res.get('data', '').startswith('Diff chunks: '):
continue
if res.get('code', '') == 'error':
errs.append(res)
continue
if 'resolveFlag' in res:
# TODO: resolveFlag can be ...?
# m: merge
# c: copy from (not conflict!)
# We skip this entry as it is the second returned from p4
# for one input file
continue
res_str = res.get('depotFile') or res.get('fromFile')
if not res_str and res.get('data'):
res_str = head + '/' + res['data']
if not res_str:
errs.append(res)
continue
res_str = Pyforce.unescape(res_str)
for i, f in enumerate(fstats):
if f"{head}/{f[F_PATH]}" in res_str:
repeats[f"{head}/{f[F_PATH]}"].append(res)
not_yet.append(fstats.pop(i))
break
else:
for f in repeats.keys():
if f in res_str:
if debug:
err_print(
f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}"
)
break
else:
if debug:
err_print("*** DEBUG: ERRS APPEND", res)
errs.append(res)
if errs:
raise LogAndAbort('Unexpected reply from p4')
if len(p4paths) == len(fstats):
raise LogAndAbort('Nothing recognized from p4')
except P4Error as e:
non_recoverable = False
for a in e.args:
if 'clobber writable file' in a['data']:
fname = a['data'].split('clobber writable file')[1].strip()
print("*** WARNING: Saving writable file as .bak:",
fname,
file=sys.stderr)
if os.path.exists(fname + '.bak'):
now = time.time()
print(
f"*** WARNING: Moved previous .bak to {fname}.{now}",
file=sys.stderr)
os.rename(fname + '.bak', f'{fname}.bak.{now}')
shutil.copy(fname, fname + '.bak')
os.chmod(fname, 0o400)
else:
non_recoverable = True
if non_recoverable:
raise
except P4TimeoutError as e:
e = str(e).replace('\n', ' ')
print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr)
retries -= 1
if not retries:
sys.exit(
f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}")
except LogAndAbort as e:
import json
fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}'
d = {
'args': args,
'fstats': fstats,
'errs': errs,
'repeats': repeats,
'infos': infos,
}
json.dump(d, open(f'.o4/{fname}', 'wt'))
sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}')
finally:
if not quiet:
for fstat in not_yet:
# Printing the fstats after the p4 process has ended, because p4 marshals
# its objects before operation, as in "And for my next act... !"
# This premature printing leads to false checksum errors during sync.
print(fstat_join(fstat))
def _writes(self, config: str, extension: str):
f = NamedTemporaryFile(mode='r+', suffix='.' + extension)
f.write(config)
f.seek(0)
return f
def get_file_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False):
content = get_content_from_s3(s3_bucket, path, conn_name, decrypt)
temp_file = NamedTemporaryFile()
temp_file.write(content)
temp_file.seek(0)
return temp_file
def temp_imagefile(width, height, format):
i = CsvExportTest.image(width, height)
f = NamedTemporaryFile(suffix='.jpg')
i.save(f, format)
f.seek(0)
return f
class ParallelUclustConsensusTaxonomyAssignerTests(TestCase):
def setUp(self):
"""
"""
self.files_to_remove = []
self.dirs_to_remove = []
tmp_dir = get_qiime_temp_dir()
self.test_out = get_tmp_filename(
tmp_dir=tmp_dir,
prefix='qiime_parallel_taxonomy_assigner_tests_',
suffix='',
result_constructor=str)
self.dirs_to_remove.append(self.test_out)
create_dir(self.test_out)
self.tmp_seq_filepath = get_tmp_filename(
tmp_dir=self.test_out,
prefix='qiime_parallel_taxonomy_assigner_tests_input',
suffix='.fasta')
seq_file = open(self.tmp_seq_filepath, 'w')
seq_file.write(uclust_test_seqs.toFasta())
seq_file.close()
self.files_to_remove.append(self.tmp_seq_filepath)
self.id_to_taxonomy_file = NamedTemporaryFile(
prefix='qiime_parallel_taxonomy_assigner_tests_id_to_taxonomy',
suffix='.txt',
dir=tmp_dir)
self.id_to_taxonomy_file.write(uclust_id_to_taxonomy)
self.id_to_taxonomy_file.seek(0)
self.reference_seqs_file = NamedTemporaryFile(
prefix='qiime_parallel_taxonomy_assigner_tests_ref_seqs',
suffix='.fasta',
dir=tmp_dir)
self.reference_seqs_file.write(uclust_reference_seqs.toFasta())
self.reference_seqs_file.seek(0)
initiate_timeout(60)
def tearDown(self):
""" """
disable_timeout()
remove_files(self.files_to_remove)
# remove directories last, so we don't get errors
# trying to remove files which may be in the directories
for d in self.dirs_to_remove:
if exists(d):
rmtree(d)
def test_parallel_uclust_taxonomy_assigner(self):
""" parallel_uclust_taxonomy_assigner functions as expected """
params = {
'id_to_taxonomy_fp': self.id_to_taxonomy_file.name,
'reference_seqs_fp': self.reference_seqs_file.name,
'uclust_min_consensus_fraction': 0.51,
'uclust_similarity': 0.90,
'uclust_max_accepts': 3
}
app = ParallelUclustConsensusTaxonomyAssigner()
r = app(self.tmp_seq_filepath,
self.test_out,
params,
job_prefix='UTATEST',
poll_directly=True,
suppress_submit_jobs=False)
results = fields_to_dict(
open(glob(join(self.test_out, '*_tax_assignments.txt'))[0], 'U'))
# some basic sanity checks: we should get the same number of sequences
# as our input with the same seq IDs. We should have a taxonomy string
# and a confidence value for each seq as well.
self.assertEqual(len(results), 6)
self.assertEqual(len(results['s1']), 3)
self.assertEqual(len(results['s6']), 3)
class Nitter(DockerBase):
"""Nitter Docker container object
Args:
host (IPv4Address): The host address the docker container will bind too.
port (int): The port the docker container will listen to.
Attributes:
tempfile (TemporaryFile): A TemporaryFile file generated from a template.
container (Container): Local representation of a container object.
Holds the started instance of a docker container.
address (str): The full address of the docker container.
ports (dict[int, int]): Binds the listening port to the nitter docker container's
internal port 8080.
config_filepath (str): Path name to the generated tempfile.
volumes (dict[str, dict[str, str]]): used to configure a bind volume.
"""
host: IPv4Address
port: int
tempfile: TemporaryFile = None
container: Optional[Container]
class Config:
arbitrary_types_allowed = True
@property
def address(self):
return f"http://{self.host}:{self.port}"
@property
def config_filepath(self):
if self.tempfile:
return self.tempfile.name
@property
def ports(self):
return {8080: self.port}
@property
def volumes(self):
volumes = {"bind": "/src/nitter.conf", "mode": "ro"}
return {self.config_filepath: volumes}
def _render_config(self):
env = Environment(loader=FileSystemLoader(TEMPLATES_DIRECTORY))
template = env.get_template("nitter.conf")
return template.render(self.dict())
def _create_configfile(self):
config = self._render_config()
self.tempfile = NamedTemporaryFile(dir=PROJECT_ROOT)
self.tempfile.write(config.encode())
self.tempfile.seek(0)
def get_profile(self, username: str, not_found_ok: bool = False):
"""Scrapes nitter for the target users profile information.
This is a modified version of nitter_scraper.profile.get_profile().
This version automatically uses the address of the docker container as the primary
address to scrape profile data.
Args:
username: The target profiles username.
not_found_ok: If not_found_ok is false (the default), a ValueError is raised if
the target profile doesn't exist. If not_found_ok is true, None will be returned
instead.
Returns:
Profile object if successfully scraped, otherwise None.
Raises:
ValueError: If the target profile does not exist and the not_found_ok argument is
false.
"""
return get_profile(username=username, not_found_ok=not_found_ok, address=self.address)
def get_tweets(self, username: str, pages: int = 25, break_on_tweet_id: Optional[int] = None):
"""Gets the target users tweets
This is a modified version of nitter_scraper.tweets.get_tweets().
This version automatically uses the address of the docker container as the primary
address to scrape profile data.
Args:
username: Targeted users username.
pages: Max number of pages to lookback starting from the latest tweet.
break_on_tweet_id: Gives the ability to break out of a loop if a tweets id is found.
address: The address to scrape from. The default is https://nitter.net which should
be used as a fallback address.
Yields:
Tweet Objects
"""
return get_tweets(
username=username,
pages=pages,
break_on_tweet_id=break_on_tweet_id,
address=self.address,
)
def start(self):
"""Starts the docker the container"""
self._create_configfile()
client = self._get_client()
self.container = client.containers.run(
image="zedeus/nitter:2c6cabb4abe79166ce9973d8652fb213c1b0c5a2",
auto_remove=True,
ports=self.ports,
detach=True,
volumes=self.volumes,
)
time.sleep(1)
logger.info(f"Running container {self.container.name} {self.container.short_id}.")
def stop(self):
"""Stops the docker the container"""
logger.info(f"Stopping container {self.container.name} {self.container.short_id}.")
if self.container:
self.container.stop(timeout=5)
logger.info(f"Container {self.container.name} {self.container.short_id} Destroyed.")
ws = wb.active
ws.title = '워크시트1'
print(wb.sheetnames)
# 10행 10열에 데이터 넣기
for x in range(1, 11):
for y in range(1, 11):
ws.cell(row=x, column=y, value=random.randint(0, 100))
print("~" * 80)
# 저장하기
# wb.save("ex09_1.xlsx")
# 예를 들어 Pyramid, FlaskExam 또는 Django와 같은 웹 애플리케이션을 사용할 때
# 파일을 스트림에 저장하려면 다음을 제공하면됩니다 NamedTemporaryFile()
tmp = None
try:
tmp = NamedTemporaryFile()
file_name = tmp.name
print(file_name)
wb.save(file_name)
tmp.seek(0)
stream = tmp.read()
print(stream)
except PermissionError as e:
print(e)
finally:
tmp.close()
print("Bye!!!")
class ParallelRdpTaxonomyAssignerTests(TestCase):
def setUp(self):
""" """
self.files_to_remove = []
self.dirs_to_remove = []
tmp_dir = get_qiime_temp_dir()
self.test_out = get_tmp_filename(
tmp_dir=tmp_dir,
prefix='qiime_parallel_taxonomy_assigner_tests_',
suffix='',
result_constructor=str)
self.dirs_to_remove.append(self.test_out)
create_dir(self.test_out)
# Temporary input file
self.tmp_seq_filepath = get_tmp_filename(
tmp_dir=self.test_out,
prefix='qiime_parallel_taxonomy_assigner_tests_input',
suffix='.fasta')
seq_file = open(self.tmp_seq_filepath, 'w')
seq_file.write(rdp_test_seqs)
seq_file.close()
self.files_to_remove.append(self.tmp_seq_filepath)
self.id_to_taxonomy_file = NamedTemporaryFile(
prefix='qiime_parallel_taxonomy_assigner_tests_id_to_taxonomy',
suffix='.txt',
dir=tmp_dir)
self.id_to_taxonomy_file.write(rdp_id_to_taxonomy)
self.id_to_taxonomy_file.seek(0)
self.reference_seqs_file = NamedTemporaryFile(
prefix='qiime_parallel_taxonomy_assigner_tests_ref_seqs',
suffix='.fasta',
dir=tmp_dir)
self.reference_seqs_file.write(rdp_reference_seqs)
self.reference_seqs_file.seek(0)
jar_fp = getenv('RDP_JAR_PATH')
jar_basename = basename(jar_fp)
if '2.2' not in jar_basename:
raise ApplicationError(
"RDP_JAR_PATH does not point to version 2.2 of the "
"RDP Classifier.")
initiate_timeout(60)
def tearDown(self):
""" """
disable_timeout()
remove_files(self.files_to_remove)
# remove directories last, so we don't get errors
# trying to remove files which may be in the directories
for d in self.dirs_to_remove:
if exists(d):
rmtree(d)
def test_parallel_rdp_taxonomy_assigner(self):
""" parallel_rdp_taxonomy_assigner functions as expected """
params = {
'id_to_taxonomy_fp': self.id_to_taxonomy_file.name,
'rdp_max_memory': 1500,
'rdp_classifier_fp': getenv('RDP_JAR_PATH'),
'confidence': 0.80,
'reference_seqs_fp': self.reference_seqs_file.name
}
app = ParallelRdpTaxonomyAssigner()
r = app(self.tmp_seq_filepath,
self.test_out,
params,
job_prefix='RDPTEST',
poll_directly=True,
suppress_submit_jobs=False)
results = fields_to_dict(
open(glob(join(self.test_out, '*_tax_assignments.txt'))[0], 'U'))
# some basic sanity checks: we should get the same number of sequences
# as our input with the same seq IDs. We should have a taxonomy string
# and a confidence value for each seq as well.
self.assertEqual(len(results), 2)
self.assertEqual(len(results['X67228']), 2)
self.assertEqual(len(results['EF503697']), 2)
print("CWD: {}".format(Path.cwd()))
print("Creating temporary file.")
temp_file = NamedTemporaryFile(mode="w+", delete=False) # type: TextIOWrapper
print("Temporary file at: {}".format(temp_file.name))
print("Loading all programs' tokens")
connection = sqlite3.connect(str(database_path))
cursor = connection.cursor()
rows = cursor.execute("SELECT tokens FROM tagger").fetchall()
print("Loading programs into temporary file")
for sentence in rows:
sentence = json.loads(sentence[0])
for token in sentence: # type: str
temp_file.write(" " + token)
temp_file.seek(0)
print("Make fast-text model.")
model = fastText.train_unsupervised(
input=temp_file.name,
#lr=0.1,
epoch=500,
minCount=4,
model="skipgram",
thread=18)
# Save model
model.save_model(str(Path(storage_folder, "model.bin")))
import pandas as pd
from tempfile import NamedTemporaryFile
from os.path import getsize
np.random.seed(42)
a = np.random.randn(365, 4)
# 임시파일로 저장하고 크기 확인
tmpf = NamedTemporaryFile()
np.savetxt(tmpf, a, delimiter=',')
print("Size CSV file", getsize(tmpf.name))
# .npy포맷으로 저장해 불러오고, 형태와 크기를 확인
tmpf = NamedTemporaryFile()
np.save(tmpf, a)
tmpf.seek(0)
loaded = np.load(tmpf)
print("Shape", loaded.shape)
print("Size .npy file", getsize(tmpf.name))
# 데이터프레임의 피클 포맷으로 만들고 읽어오기
df = pd.DataFrame(a)
print(df)
df.to_pickle('tmpf.pkl')
print("Size pickled dataframe", getsize('tmpf.pkl'))
print("DF from pickle\n", pd.read_pickle('tmpf.pkl'))
## 3. PyTables와 데이터 저장
# pip install numexpr tables - HDF5 라이브러리(데이터를 그룹과 데이터셋 형태로 구조화)
import numpy as np
editor = os.environ['EDITOR']
# Open the envrionment
env = Environment('/var/trac/helpdesk')
# Grab the page model
page = WikiPage(env, name)
# Make a temporary file
safename = name.translate(string.maketrans('/', '_'))
file = NamedTemporaryFile(mode='w+', prefix=safename, suffix='.txt')
# If the page exists, populate the tempfile
if page.exists:
file.write(page.text)
file.flush()
# Open the file in $EDITOR
os.spawnlp(os.P_WAIT, editor, editor, file.name)
# Reread the text
file.seek(0)
page.text = file.read()
# Save the file back
try:
page.save(author=os.getlogin(), comment='', remote_addr='127.0.0.1')
print 'Page changed succesfully'
except TracError, e:
print 'Error: %s' % e.message
class Converter:
def __init__(self, version="", shapes=False, clear_shape_errors=True):
clear_directory("gtfs")
if clear_shape_errors: clear_directory("shape-errors")
# Stop info
self.missing_stops = requests.get(
"https://gist.githubusercontent.com/MKuranowski/0ca97a012d541899cb1f859cd0bab2e7/raw/missing_stops.json"
).json()
self.rail_platforms = requests.get(
"https://gist.githubusercontent.com/MKuranowski/0ca97a012d541899cb1f859cd0bab2e7/raw/rail_platforms.json"
).json()
self.incorrect_stops = []
self.unused_stops = list(self.missing_stops.keys())
self.stops_map = {}
self.stop_names = PROPER_STOP_NAMES.copy()
# File handler
self.version = None
self.reader = None
self.parser = None
# Get shape generator instance
if isinstance(shapes, Shaper):
self.shapes = shapes
self.shapes.open()
elif shapes:
self.shapes = Shaper()
self.shapes.open()
else:
self.shapes = None
self.get_file(version)
def get_file(self, version):
"Download and decompress schedules for current data. Returns tuple (TemporaryFile, version) - and that TemporaryFile is decompressed .TXT file"
# Login to ZTM server and get the list of files
server = FTP("rozklady.ztm.waw.pl")
server.login()
files = [f for f in server.nlst() if re.fullmatch(r"RA\d{6}\.7z", f)]
# If user has requested an exact version, check if it's on the server
if version:
fname = "{}.7z".format(version)
if fname not in files:
raise KeyError(
"Requested file version ({}) not found on ZTM server".
format(version))
# If not, find one valid today
else:
fdate = date.today()
while True:
fname = fdate.strftime("RA%y%m%d.7z")
if fname in files: break
else: fdate -= timedelta(days=1)
# Create temporary files for storing th 7z archive and the compressed TXT file
temp_arch = NamedTemporaryFile(mode="w+b", delete=False)
self.reader = NamedTemporaryFile(mode="w+t", delete=True)
try:
# Download the file
server.retrbinary("RETR " + fname, temp_arch.write)
server.quit()
temp_arch.close()
# Open the temporary archive inside
with libarchive.public.file_reader(temp_arch.name) as arch:
# Iterate over each file inside the archive
for arch_file in arch:
# Assert the file inside the archive is the TXT file we're looking for
name_match = re.fullmatch(r"(RA\d{6})\.TXT",
arch_file.pathname,
flags=re.IGNORECASE)
if not name_match:
continue
# Save the feed version
self.version = name_match[1].upper()
# Decompress the TXT file block by block and save it to the reader
for block in arch_file.get_blocks():
self.reader.write(str(block, "cp1250"))
self.reader.seek(0)
# only one TXT file should be inside the archive
break
else:
raise FileNotFoundError(
"no schedule file found inside archive {}".format(
fname))
# Remove the temp arch file at the end
finally:
os.remove(temp_arch.name)
self.parser = Parser(self.reader)
def calendar(self):
file = open("gtfs/calendar_dates.txt",
mode="w",
encoding="utf8",
newline="")
writer = csv.writer(file)
writer.writerow(["service_id", "date", "exception_type"])
print("\033[1A\033[K" + "Parsing calendars (KA)")
for day in self.parser.parse_ka():
for service_id in day["services"]:
writer.writerow([service_id, day["date"], "1"])
file.close()
def _stopgroup_railway(self, writer, group_id, group_name):
# Load ZTM stakes from PR section
# self.parser.parse_pr() has to be called to skip to the next entry in ZP
stakes = list(self.parser.parse_pr())
# If group is not in ACTIVE_RAIL_STATIONS, ignore it
if group_id not in ACTIVE_RAIL_STATIONS:
for s in stakes:
self.stops_map[s["id"]] = None
return
# Basic info about the station
station_info = self.rail_platforms.get(group_id, {})
# If this station is not in rail_platforms, average all stake positions
# In order to calculate an approx position of the station
if not station_info:
stake_positions = [(i["lat"], i["lon"]) for i in stakes]
stake_positions = [i for i in stake_positions if i[0] and i[1]]
if stake_positions:
station_lat, station_lon = avg_position(stake_positions)
# No position for the station
else:
for s in stakes:
self.stops_map[s["id"]] = None
self.incorrect_stops.append(group_id)
return
# Otherwise get the position from rail_platforms data
else:
station_lat, station_lon = map(float,
station_info["pos"].split(","))
group_name = station_info["name"]
# One Platform or No Platform data
if (not station_info) or station_info["oneplatform"]:
# Save position for shapes
if self.shapes:
self.shapes.stops[group_id] = station_lat, station_lon
# Add info for stops_map
for stake in stakes:
self.stops_map[stake["id"]] = group_id
# Output info to GTFS
writer.writerow([
group_id,
group_name,
station_lat,
station_lon,
"",
"",
station_info.get("ibnr_code", ""),
"",
station_info.get("wheelchair", 0),
])
# Multi-Platform station
else:
# Hub entry
writer.writerow([
group_id,
group_name,
station_lat,
station_lon,
"1",
"",
station_info["ibnr_code"],
"",
station_info.get("wheelchair", 0),
])
# Platforms
for platform_id, platform_pos in station_info["platforms"].items():
platform_lat, platform_lon = map(float,
platform_pos.split(","))
platform_code = platform_id.split("p")[1]
platform_name = f"{group_name} peron {platform_code}"
# Save position for shapes
if self.shapes:
self.shapes.stops[platform_id] = platform_lat, platform_lon
# Output to GTFS
writer.writerow([
platform_id,
platform_name,
platform_lat,
platform_lon,
"0",
group_id,
station_info["ibnr_code"],
platform_code,
station_info.get("wheelchair", 0),
])
# Stops → Platforms
for stake in stakes:
# Defined stake in rail_platforms
if stake["id"] in station_info["stops"]:
self.stops_map[stake["id"]] = station_info["stops"][
stake["id"]]
# Unknown stake
elif stake["id"] not in {"491303", "491304"}:
warn(
f'No platform defined for railway PR entry {group_name} {stake["id"]}'
)
def _stopgroup_normal(self, writer, group_id, group_name):
# Load ZTM stakes from PR section
# self.parser.parse_pr() has to be called to skip to the next entry in ZP
stakes = list(self.parser.parse_pr())
# Split virtual stakes from normal stakes
virtual_stakes = [i for i in stakes if i["code"][0] == "8"]
normal_stakes = [i for i in stakes if i["code"][0] != "8"]
# Load positions from missing_stops to normal_stakes
for idx, stake in enumerate(normal_stakes):
if (stake["lat"] == None or stake["lon"] == None) and \
stake["id"] in self.missing_stops:
self.unused_stops.remove(stake["id"])
stake["lat"], stake["lon"] = self.missing_stops[stake["id"]]
normal_stakes[idx] = stake
position_stakes = [i for i in normal_stakes if i["lat"] and i["lon"]]
# Convert normal stakes
for stake in normal_stakes:
# Position defined
if stake["lat"] and stake["lon"]:
# Save position for shapes
if self.shapes:
self.shapes.stops[stake["id"]] = stake["lat"], stake["lon"]
# Output info to GTFS
writer.writerow([
stake["id"],
f'{group_name} {stake["code"]}',
stake["lat"],
stake["lon"],
"",
"",
"",
"",
stake["wheelchair"],
])
# Position undefined
else:
self.stops_map[stake["id"]] = None
self.incorrect_stops.append(stake["id"])
# Convert virtual stops
for stake in virtual_stakes:
stakes_with_same_pos = [i["id"] for i in position_stakes if \
(i["lat"], i["lon"]) == (stake["lat"], stake["lon"])]
stakes_with_same_code = [i["id"] for i in position_stakes if \
i["code"][1] == stake["code"][1]]
# Metro Młociny 88 → Metro Młociny 28
if stake["id"] == "605988":
counterpart_available = [i for i in position_stakes if \
i["id"] == "605928"]
# If 605928 is present, map 605988 to it.
# Otherwise fallback on defualt maching options
if counterpart_available:
self.stops_map["605988"] = "605928"
continue
# Map to a stake with same position
if stakes_with_same_pos:
self.stops_map[stake["id"]] = stakes_with_same_pos[0]
# Map to a stake with same digit
elif stakes_with_same_code:
self.stops_map[stake["id"]] = stakes_with_same_code[0]
# Unable find a matching stake
else:
self.stops_map[stake["id"]] = None
self.incorrect_stops.append(stake["id"])
def stops(self):
file = open("gtfs/stops.txt", mode="w", encoding="utf8", newline="")
writer = csv.writer(file)
writer.writerow([
"stop_id", "stop_name", "stop_lat", "stop_lon", "location_type",
"parent_station", "stop_IBNR", "platform_code",
"wheelchair_boarding"
])
print("\033[1A\033[K" + "Parsing stops (ZP)")
for group in self.parser.parse_zp():
# Fix town name for Kampinoski PN
if group["town"] == "Kampinoski Pn":
group["town"] = "Kampinoski PN"
# Add name to self.stop_names if it's missing
if group["id"] not in self.stop_names:
group["name"] = normal_stop_name(group["name"])
self.stop_names[group["id"]] = group["name"]
else:
group["name"] = self.stop_names[group["id"]]
# Add town name to stop name
if should_town_be_added_to_name(group):
group["name"] = f'{group["town"]} {group["name"]}'
self.stop_names[group["id"]] = group["name"]
# Parse stakes
if group["id"][1:3] in {"90", "91", "92"}:
self._stopgroup_railway(writer, group["id"], group["name"])
else:
self._stopgroup_normal(writer, group["id"], group["name"])
file.close()
def routes_schedules(self):
file_routes = open("gtfs/routes.txt",
mode="w",
encoding="utf8",
newline="")
writer_routes = csv.writer(file_routes)
writer_routes.writerow([
"agency_id", "route_id", "route_short_name", "route_long_name",
"route_type", "route_color", "route_text_color", "route_sort_order"
])
file_trips = open("gtfs/trips.txt",
mode="w",
encoding="utf8",
newline="")
writer_trips = csv.writer(file_trips)
writer_trips.writerow([
"route_id", "service_id", "trip_id", "trip_headsign",
"direction_id", "shape_id", "exceptional", "wheelchair_accessible",
"bikes_allowed"
])
file_times = open("gtfs/stop_times.txt",
mode="w",
encoding="utf8",
newline="")
writer_times = csv.writer(file_times)
writer_times.writerow([
"trip_id", "arrival_time", "departure_time", "stop_id",
"stop_sequence", "pickup_type", "drop_off_type",
"shape_dist_traveled"
])
route_sort_order = 1 # Leave first 2 blank for M1 and M2 routes
route_id = None
print("\033[1A\033[K" + "Parsing routes & schedules (LL)")
for route in self.parser.parse_ll():
route_id, route_desc = route["id"], route["desc"]
# Ignore Koleje Mazowieckie & Warszawska Kolej Dojazdowa routes
if route_id.startswith("R") or route_id.startswith("WKD"):
self.parser.skip_to_section("WK", end=True)
continue
print("\033[1A\033[K" +
f"Parsing routes & schedules (LL) - {route_id}")
route_sort_order += 1
route_type, route_color, route_text_color = route_color_type(
route_id, route_desc)
# Data loaded from TR section
route_name = ""
direction_stops = {"0": set(), "1": set()}
on_demand_stops = set()
inaccesible_trips = set()
variant_directions = {}
# Variants
print("\033[1A\033[K" +
f"Parsing routes & schedules (TR) - {route_id}")
for variant in self.parser.parse_tr():
print("\033[1A\033[K" +
f"Parsing routes & schedules (LW) - {route_id}")
stops = list(self.parser.parse_lw())
# variant direction
variant_directions[variant["id"]] = variant["direction"]
# route_name should be the name of first and last stop of 1st variant
if not route_name:
route_name = " — ".join([
self.stop_names[stops[0]["id"][:4]],
self.stop_names[stops[-1]["id"][:4]]
])
# add on_demand_stops from this variant
on_demand_stops |= {i["id"] for i in stops if i["on_demand"]}
# add stopids to proper direction in direction_stops
direction_stops[variant["direction"]] |= {
i["id"]
for i in stops
}
# now parse ODWG sections - for inaccesible trips (only tram)
if route_type == "0":
print("\033[1A\033[K" +
f"Parsing routes & schedules (TD) - {route_id}")
for trip in self.parser.parse_wgod(route_type, route_id):
if not trip["accessible"]:
inaccesible_trips.add(trip["id"])
else:
self.parser.skip_to_section("RP", end=True)
# Schedules
print("\033[1A\033[K" +
f"Parsing routes & schedules (WK) - {route_id}")
for trip in self.parser.parse_wk(route_id):
# Change stop_ids based on stops_map
for stopt in trip["stops"]:
stopt["orig_stop"] = stopt.pop("stop")
stopt["stop"] = self.stops_map.get(stopt["orig_stop"],
stopt["orig_stop"])
# Fliter "None" stops
trip["stops"] = [i for i in trip["stops"] if i["stop"]]
# Ignore trips with only 1 stopt
if len(trip["stops"]) < 2: continue
# Unpack info from trip_id
trip_id = trip["id"]
trip_id_split = trip_id.split("/")
variant_id = trip_id_split[1]
service_id = trip_id_split[2]
del trip_id_split
# "Exceptional" trip - a deutor/depot run
if variant_id.startswith("TP-") or variant_id.startswith(
"TO-"):
exceptional = "0"
else:
exceptional = "1"
# Shapes
if self.shapes:
shape_id, shape_distances = self.shapes.get(
route_type, trip_id,
[i["stop"] for i in trip["stops"]])
else:
shape_id, shape_distances = "", {}
# Wheelchair Accessibility
if trip_id in inaccesible_trips:
wheelchair = "2"
else:
wheelchair = "1"
# Direction
if variant_id in variant_directions:
direction = variant_directions[variant_id]
else:
direction = trip_direction(
{i["orig_stop"]
for i in trip["stops"]}, direction_stops)
variant_directions[variant_id] = direction
# Headsign
headsign = proper_headsign(
trip["stops"][-1]["stop"],
self.stop_names.get(trip["stops"][-1]["stop"][:4], ""))
if not headsign:
warn(f"No headsign for trip {trip_id}")
# Write to trips.txt
writer_trips.writerow([
route_id,
service_id,
trip_id,
headsign,
direction,
shape_id,
exceptional,
wheelchair,
"1",
])
max_seq = len(trip["stops"]) - 1
# StopTimes
for seq, stopt in enumerate(trip["stops"]):
# Pickup Type
if seq == max_seq: pickup = "1"
elif "P" in stopt["flags"]: pickup = "1"
elif stopt["orig_stop"] in on_demand_stops: pickup = "3"
else: pickup = "0"
# Drop-Off Type
if seq == 0: dropoff = "1"
elif stopt["orig_stop"] in on_demand_stops: dropoff = "3"
else: dropoff = "0"
# Shape Distance
stop_dist = shape_distances.get(seq, "")
if stop_dist: stop_dist = round(stop_dist, 4)
# Output to stop_times.txt
writer_times.writerow([
trip_id, stopt["time"], stopt["time"], stopt["stop"],
seq, pickup, dropoff, stop_dist
])
# Output to routes.txt
writer_routes.writerow([
"0", route_id, route_id, route_name, route_type, route_color,
route_text_color, route_sort_order
])
file_routes.close()
file_trips.close()
file_times.close()
def parse(self):
self.calendar()
self.stops()
self.routes_schedules()
def dump_missing_stops(self):
with open("missing_stops.json", "w") as f:
json.dump(
{
"missing": [int(i) for i in self.incorrect_stops],
"unused": [int(i) for i in self.unused_stops],
},
f,
indent=0)
@staticmethod
def static_files(shapes, version, download_time):
feed_version = "Version {}; downloaded at: {}".format(
version, download_time)
"Create files that don't depend of ZTM file content"
file = open("gtfs/agency.txt",
mode="w",
encoding="utf8",
newline="\r\n")
file.write(
'agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_fare_url\n'
)
file.write(
'0,"Warszawski Transport Publiczny","https://wtp.waw.pl",Europe/Warsaw,pl,19 115,"https://www.nowa.wtp.waw.pl/ceny-i-rodzaje-biletow/"\n'
)
file.close()
file = open("gtfs/feed_info.txt",
mode="w",
encoding="utf8",
newline="\r\n")
file.write(
'feed_publisher_name,feed_publisher_url,feed_lang,feed_version\n')
if shapes:
file.write(
'"GTFS Convert: MKuranowski; Data: ZTM Warszawa; Bus Shapes (under ODbL License): © OpenStreetMap contributors","https://github.com/MKuranowski/WarsawGTFS",pl,{}\n'
.format(feed_version))
else:
file.write(
'"GTFS Convert: MKuranowski; Data: ZTM Warszawa","https://github.com/MKuranowski/WarsawGTFS",pl,{}\n'
.format(feed_version))
file.close()
@staticmethod
def compress(target="gtfs.zip"):
"Compress all created files to gtfs.zip"
with zipfile.ZipFile(target,
mode="w",
compression=zipfile.ZIP_DEFLATED) as archive:
for file in os.listdir("gtfs"):
if file.endswith(".txt"):
archive.write(os.path.join("gtfs", file), arcname=file)
@classmethod
def create(cls,
version="",
shapes=False,
metro=False,
prevver="",
targetfile="gtfs.zip",
clear_shape_errors=True):
print("\033[1A\033[K" + "Downloading file")
download_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S")
self = cls(version, shapes)
if prevver == self.version:
self.reader.close()
print("\033[1A\033[K" +
"File matches the 'prevver' argument, aborting!")
return
print("\033[1A\033[K" + "Starting parser...")
self.parse()
print("\033[1A\033[K" + "Parser finished working, closing TXT file")
self.reader.close()
self.dump_missing_stops()
print("\033[1A\033[K" + "Creating static files")
self.static_files(bool(self.shapes), self.version, download_time)
if metro:
print("\033[1A\033[K" + "Adding metro")
Metro.add()
print("\033[1A\033[K" + "Compressing")
self.compress(targetfile)
return self.version
def post(self, request):
file_content = None
tmp_file = None
try:
file_content = request.raw_post_data
file_errors, file_extension = _perform_file_validations(request)
tmp_file = NamedTemporaryFile(delete=True, suffix=file_extension)
if file_errors:
logger.info(
"User: %s. Upload File validation failed: %s. File name: %s, size: %d",
request.user.username, json.dumps(file_errors),
request.GET.get("qqfile"),
int(request.META.get('CONTENT_LENGTH')))
return HttpResponse(json.dumps({
'success': False,
'error_msg': file_errors
}),
content_type='application/json')
tmp_file.write(file_content)
tmp_file.seek(0)
project_name = request.GET['pname'].strip()
manager = get_database_manager(request.user)
questionnaire_code = generate_questionnaire_code(manager)
xls_parser_response = XlsFormParser(tmp_file, project_name,
manager).parse()
if xls_parser_response.errors:
error_list = list(xls_parser_response.errors)
logger.info("User: %s. Upload Errors: %s",
request.user.username, json.dumps(error_list))
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg':
error_list,
'message_prefix':
_("Sorry! Current version of DataWinners does not support"
),
'message_suffix':
_("Update your XLSForm and upload again.")
}))
tmp_file.seek(0)
mangrove_service = MangroveService(
request,
questionnaire_code=questionnaire_code,
project_name=project_name,
xls_form=tmp_file,
xls_parser_response=xls_parser_response)
questionnaire_id, form_code = mangrove_service.create_project()
except PyXFormError as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
message = transform_error_message(e.message)
if 'name_type_error' in message or 'choice_name_type_error' in message:
if 'choice_name_type_error' in message:
message_prefix = _(
"On your \"choices\" sheet the first and second column must be \"list_name\" and \"name\". Possible errors:"
)
else:
message_prefix = _(
"On your \"survey\" sheet the first and second column must be \"type\" and \"name\". Possible errors:"
)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Columns are missing"),
_("Column name is misspelled"),
_("Additional space in column name")
],
'message_prefix':
message_prefix,
'message_suffix':
_("Update your XLSForm and upload again.")
}))
else:
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
message if message else ugettext(
"all XLSForm features. Please check the list of unsupported features."
)
]
}))
except QuestionAlreadyExistsException as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Duplicate labels. All questions (labels) must be unique."
)
],
'message_prefix':
_("Sorry! Current version of DataWinners does not support"
),
'message_suffix':
_("Update your XLSForm and upload again.")
}))
except UnicodeDecodeError as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Check your columns for errors.<br>There are missing symbols (like $ for relevant or calculate) or incorrect characters<br>"
) + _("Update your XLSForm and upload again.")
],
}))
except Exception as e:
message = e.message if e.message else _("Errors in excel")
logger.info("User: %s. Upload Exception message: %s",
request.user.username, e.message)
odk_message = ''
if not 'ODK Validate Errors:' in e.message:
send_email_on_exception(
request.user,
"Questionnaire Create",
traceback.format_exc(),
additional_details={'file_contents': file_content})
else:
odk_message = translate_odk_message(e.message)
message = odk_message if odk_message else message
return HttpResponse(content_type='application/json',
content=json.dumps({
'success': False,
'error_msg': [message],
}))
finally:
if tmp_file:
tmp_file.close()
if not questionnaire_id:
return HttpResponse(json.dumps({
'success':
False,
'duplicate_project_name':
True,
'error_msg': [
_("Questionnaire with same name already exists.Upload was cancelled."
)
]
}),
content_type='application/json')
return HttpResponse(json.dumps({
"success": True,
"project_name": project_name,
"project_id": questionnaire_id,
"form_code": form_code
}),
content_type='application/json')
def main(parser: COP, options: 'Values', reg: str) -> None:
workflow, flow_file = parse_reg(reg, src=True)
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
# read in the flow.cylc file
viewcfg = {
'mark': options.mark,
'single': options.single,
'label': options.label,
'empy': options.empy or options.process,
'jinja2': options.jinja2 or options.process,
'contin': options.cat or options.process,
'inline': (options.inline or options.jinja2 or options.empy
or options.process),
}
lines = read_and_proc(
flow_file,
load_template_vars(options.templatevars, options.templatevars_file),
viewcfg=viewcfg)
if options.stdout:
for line in lines:
print(line)
sys.exit(0)
# write to a temporary file
viewfile = NamedTemporaryFile(
suffix=".flow.cylc", prefix=workflow.replace('/', '_') + '.',
)
for line in lines:
viewfile.write((line + '\n').encode())
viewfile.seek(0, 0)
# set the file to be read only
os.chmod(viewfile.name, 0o400)
# capture the temp file's mod time in case the user edits it
# and overrides the readonly mode.
modtime1 = os.stat(viewfile.name).st_mtime
# in case editor has options, e.g. 'emacs -nw':
command_list = shlex.split(editor)
command_list.append(viewfile.name)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
# !!!VIEWING FINISHED!!!
# Did the user edit the file
modtime2 = os.stat(viewfile.name).st_mtime
if modtime2 > modtime1:
print(
"\nWARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY COPY "
f"OF THE WORKFLOW:\n {viewfile.name}\n",
file=sys.stderr
)
# DONE
viewfile.close()
def temp_imagefile(width, height, format):
i = image(width, height)
f = NamedTemporaryFile(suffix=IMG_SUFFIX[format])
i.save(f, format)
f.seek(0)
return f
if len(item_rows):
rows.append(order_row)
rows.append(date_row)
rows.append('')
rows.append(item_header)
for row in item_rows:
rows.append(row)
rows.append(total_row)
rows.append('')
rows.append('')
fp = NamedTemporaryFile(suffix=".csv")
writer = UnicodeWriter(fp)
writer.writerows(rows)
fp.seek(0)
output = fp.read()
# Send file to Mimic as attachment
body = 'Attached is the Trillium report for orders placed during the week ending %s.' % friday.strftime(
"%B %d, %Y")
subject = '[Mimic OOS] Weekly Trillium report'
email = EmailMessage(
subject,
body,
'*****@*****.**',
[
'*****@*****.**',
],
)
email.attach_file(fp.name)
def create_invoice_excel(reference_invoice):
package_path = os.path.dirname(__file__) + '/..'
invoice_wb = openpyxl.open(f'{package_path}/templates/invoice_template.xlsx')
invoice_dict = reference_invoice.to_dict()
order_products = get_invoice_order_products(reference_invoice)
total = reduce(lambda acc, op: acc + op['subtotal'], order_products, 0)
ws = invoice_wb.worksheets[0]
pl = invoice_wb.worksheets[1]
# Set invoice header
ws.cell(7, 2, reference_invoice.id)
ws.cell(7, 5, reference_invoice.when_created)
ws.cell(13, 4, reference_invoice.customer)
ws.cell(15, 2, reference_invoice.payee)
ws.cell(17, 4, reference_invoice.orders[0].address)
ws.cell(21, 4, '') # city
ws.cell(23, 5, reference_invoice.orders[0].country.name)
ws.cell(25, 4, reference_invoice.orders[0].phone)
# Set packing list header
pl.cell(7, 2, reference_invoice.id)
pl.cell(7, 5, reference_invoice.when_created)
pl.cell(13, 4, reference_invoice.customer)
ws.cell(15, 2, reference_invoice.payee)
pl.cell(17, 4, reference_invoice.orders[0].address)
pl.cell(21, 4, '') # city
pl.cell(23, 5, reference_invoice.orders[0].country.name)
pl.cell(25, 4, reference_invoice.orders[0].phone)
# Set invoice footer
ws.cell(305, 5, total)
ws.cell(311, 4, f"{round(total, 2)} USD")
ws.cell(312, 2, f"{invoice_dict['weight']}g")
# Set packing list footer
pl.cell(311, 4, f"{reduce(lambda qty, op: qty + op['quantity'], order_products, 0)}psc")
pl.cell(312, 2, f"{invoice_dict['weight']}g")
# Set order product lines
row = 31
last_row = 304
for op in order_products:
# Set invoice product item
ws.cell(row, 1, op['id'])
ws.cell(row, 2, op['name'])
ws.cell(row, 3, op['quantity'])
ws.cell(row, 4, op['price'])
ws.cell(row, 5, op['subtotal'])
# Set packing list product item
pl.cell(row, 1, op['id'])
pl.cell(row, 2, op['name'])
pl.cell(row, 4, op['quantity'])
row += 1
ws.delete_rows(row, last_row - row + 1)
pl.delete_rows(row, last_row - row + 1)
file = NamedTemporaryFile()
invoice_wb.save(file.name)
file.seek(0)
return file
def post(self, request, project_id):
manager = get_database_manager(request.user)
questionnaire = Project.get(manager, project_id)
file_content = None
tmp_file = None
try:
file_content = request.raw_post_data
file_errors, file_extension = _perform_file_validations(request)
tmp_file = NamedTemporaryFile(delete=True, suffix=file_extension)
if file_errors:
logger.info(
"User: %s. Edit upload File validation failed: %s. File name: %s, size: %d",
request.user.username, json.dumps(file_errors),
request.GET.get("qqfile"),
int(request.META.get('CONTENT_LENGTH')))
return HttpResponse(content_type='application/json',
content=json.dumps({
'success': False,
'error_msg': file_errors
}))
tmp_file.write(file_content)
tmp_file.seek(0)
xls_parser_response = XlsFormParser(tmp_file, questionnaire.name,
manager).parse()
if xls_parser_response.errors:
info_list = list(xls_parser_response.errors)
logger.info("User: %s. Edit upload Errors: %s",
request.user.username, json.dumps(info_list))
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg':
info_list,
'message_prefix':
_("Sorry! Current version of DataWinners does not support"
),
'message_suffix':
_("Update your XLSForm and upload again.")
}))
mangrove_service = MangroveService(
request,
questionnaire_code=questionnaire.form_code,
project_name=questionnaire.name,
xls_parser_response=xls_parser_response)
questionnaire.xform = mangrove_service.xform_with_form_code
QuestionnaireBuilder(questionnaire,
manager).update_questionnaire_with_questions(
xls_parser_response.json_xform_data)
tmp_file.seek(0)
questionnaire.update_media_field_flag()
questionnaire.save(process_post_update=False)
base_name, extension = os.path.splitext(tmp_file.name)
questionnaire.update_attachments(tmp_file,
'questionnaire%s' % extension)
self._purge_submissions(manager, questionnaire)
self._purge_feed_documents(questionnaire, request)
self._purge_media_details_documents(manager, questionnaire)
self.recreate_submissions_mapping(manager, questionnaire)
if xls_parser_response.info:
info_list = list(xls_parser_response.info)
logger.info("User: %s. Edit upload Errors: %s",
request.user.username, json.dumps(info_list))
return HttpResponse(content_type='application/json',
content=json.dumps({
'success':
True,
'information':
info_list,
}))
except PyXFormError as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
message = transform_error_message(e.message)
if 'name_type_error' in message or 'choice_name_type_error' in message:
if 'choice_name_type_error' in message:
message_prefix = _(
"On your \"choices\" sheet the first and second column must be \"list_name\" and \"name\". Possible errors:"
)
else:
message_prefix = _(
"On your \"survey\" sheet the first and second column must be \"type\" and \"name\". Possible errors:"
)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Columns are missing"),
_("Column name is misspelled"),
_("Additional space in column name")
],
'message_prefix':
message_prefix,
'message_suffix':
_("Update your XLSForm and upload again.")
}))
else:
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
message if message else ugettext(
"all XLSForm features. Please check the list of unsupported features."
)
]
}))
except QuestionAlreadyExistsException as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Duplicate labels. All questions (labels) must be unique."
)
],
'message_prefix':
_("Sorry! Current version of DataWinners does not support"
),
'message_suffix':
_("Update your XLSForm and upload again.")
}))
except UnicodeDecodeError as e:
logger.info("User: %s. Upload Error: %s", request.user.username,
e.message)
return HttpResponse(
content_type='application/json',
content=json.dumps({
'success':
False,
'error_msg': [
_("Check your columns for errors.<br>There are missing symbols (like $ for relevant or calculate) or incorrect characters<br>"
) + _("Update your XLSForm and upload again.")
],
}))
except Exception as e:
logger.info("User: %s. Edit Upload Exception message: %s",
request.user.username, e.message)
message = e.message if e.message else _("Some error in excel")
odk_message = ''
if not 'ODK Validate Errors:' in e.message:
send_email_on_exception(
request.user,
"Questionnaire Edit",
traceback.format_exc(),
additional_details={'file_contents': file_content})
else:
odk_message = translate_odk_message(e.message)
message = odk_message if odk_message else message
return HttpResponse(content_type='application/json',
content=json.dumps({
'error_msg': [message],
'success': False,
}))
finally:
if tmp_file:
tmp_file.close()
return HttpResponse(
json.dumps({
"success":
True,
"project_name":
questionnaire.name,
"project_id":
questionnaire.id,
"file_name":
"%s%s" % (slugify(questionnaire.name), extension),
# "xls_dict": XlsProjectParser().parse(file_content)
}),
content_type='application/json')