def upload_schedule(request):
csv2 = auto_schedule()
d = {
'data': auto_schedule.objects.all(),
}
if 'csv' in request.FILES:
form_data = TextIOWrapper(request.FILES['csv'].file, encoding='utf-8')
csv_file = csv.reader(form_data, skipinitialspace=True)
for line in csv_file:
csv2, created = auto_schedule.objects.get_or_create(pk=line[3])
csv2.kind = line[0]
csv2.kindofdata = line[1]
csv2.dateofsend = line[2]
csv2.dateofrace1 = line[3]
csv2.placeofrace1 = line[4]
csv2.place_code1_1 = line[5]
csv2.numberofrace1_1 = line[6]
csv2.place_code1_2 = line[7]
csv2.numberofrace1_2 = line[8]
csv2.place_code1_3 = line[9]
csv2.numberofrace1_3 = line[10]
csv2.place_code1_4 = line[11]
csv2.numberofrace1_4 = line[12]
csv2.place_code1_5 = line[13]
csv2.numberofrace1_5 = line[14]
csv2.place_code1_6 = line[15]
csv2.numberofrace1_6 = line[16]
csv2.dateofrace2 = line[17]
csv2.placeofrace2 = line[18]
csv2.place_code2_1 = line[19]
csv2.numberofrace2_1 = line[20]
csv2.place_code2_2 = line[21]
csv2.numberofrace2_2 = line[22]
csv2.place_code2_3 = line[23]
csv2.numberofrace2_3 = line[24]
csv2.place_code2_4 = line[25]
csv2.numberofrace2_4 = line[26]
csv2.place_code2_5 = line[27]
csv2.numberofrace2_5 = line[28]
csv2.place_code2_6 = line[29]
csv2.numberofrace2_6 = line[30]
csv2.dateofrace3 = line[31]
csv2.placeofrace3 = line[32]
csv2.place_code3_1 = line[33]
csv2.numberofrace3_1 = line[34]
csv2.place_code3_2 = line[35]
csv2.numberofrace3_2 = line[36]
csv2.place_code3_3 = line[37]
csv2.numberofrace3_3 = line[38]
csv2.place_code3_4 = line[39]
csv2.numberofrace3_4 = line[40]
csv2.place_code3_5 = line[41]
csv2.numberofrace3_5 = line[42]
csv2.place_code3_6 = line[43]
csv2.numberofrace3_6 = line[44]
csv2.prize30 = line[45]
csv2.save()
return render(request, 'app_autorace/upload_schedule.html', d)
else:
return render(request, 'app_autorace/upload_schedule.html', d)
yield get_float_token()
elif self.peek == '>' or self.peek == '<':
arrow = self.peek
if self.read() and self.peek == '=':
yield Token(Tag.BinaryOperator, None, arrow + "=", self.span_end())
else:
yield Token(Tag.BinaryOperator, None, arrow, self.span_end())
self.unread()
elif self.peek == '!' or self.peek == '=':
equality = self.peek
if self.read() and self.peek == '=':
yield Token(Tag.BinaryOperator, None, equality + "=", self.span_end())
else:
assert False, "Syntax Error " + \
str(self.span_end()) + ": Expected '='"
else:
yield Token(Tag.BinaryOperator, None, self.peek, self.span_end())
def scan_while(self, predicate):
if self.read():
if predicate(self.peek):
return self.peek + self.scan_while(predicate)
self.unread()
return ""
l = Lexer(TextIOWrapper(BytesIO(input().encode())))
for token in l.scan():
print(token.__str__())
def main():
import codecs
parser = argparse.ArgumentParser(description="Extract and print psm annotat" \
"ion data from LRLP in a form that is amen" \
"able to insertion into future xml",
formatter_class=\
argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--infile", "-i", nargs='+', type=argparse.FileType('rb'),
default=[sys.stdin,], help="input zip file(s)" \
" (each contains a multi file)")
parser.add_argument("--outfile",
"-o",
type=argparse.FileType('w'),
default=sys.stdout,
help="where to write extracted semantic info")
try:
args = parser.parse_args()
except IOError as msg:
parser.error(str(msg))
outfile = args.outfile
nonehash = {"value": "None"}
for infile in args.infile:
inbase = '.'.join(os.path.basename(infile.name).split('.')[:-2])
archive = zf(infile)
for info in archive.infolist():
if info.file_size < 20:
continue
# Assume psm structure
if os.path.dirname(info.filename) != 'psm':
continue
with TextIOWrapper(archive.open(info, 'r')) as ifh:
xobj = ET.parse(ifh)
try:
headlines = [(x.get("begin_offset"), x.get("char_length")) \
for x in xobj.findall("string[@type='headline']")]
# TODO: funornone this back into functional
postnodes = xobj.findall("string[@type='post']")
posts = []
for x in postnodes:
post = []
anode = x.find("attribute[@name='author']")
if anode is None:
anode = nonehash
dnode = x.find("attribute[@name='datetime']")
if dnode is None:
dnode = nonehash
posts.append(
(x.get("begin_offset"), x.get("char_length"),
anode.get('value'), dnode.get('value')))
except:
print(info.filename)
raise
sys.exit(1)
# GENRE/LANG/DATE info will be gleaned from filename later.
# assume psm.xml and strip it off
fname = os.path.basename(info.filename).split(".psm.xml")[0]
for h in headlines:
outfile.write("\t".join(("headline", fname) + h) + "\n")
for p in posts:
outfile.write("\t".join(("post", fname) + p) + "\n")
def _load_contents(bucket: str, key: str) -> TextIOWrapper:
response = _S3_CLIENT.get_object(Bucket=bucket, Key=key)
gzipped = GzipFile(None, 'rb', fileobj=response['Body'])
return TextIOWrapper(gzipped) # type: ignore
def _file_iterator(self, name):
"""DictReader iterator for a file in the gtfs archive"""
gtfs_zip = ZipFile(self.path)
with TextIOWrapper(gtfs_zip.open(name), encoding='utf8') as gtfs_csv:
yield DictReader(gtfs_csv)
ad_data["report_date"] = bundle_date
total_rows += 1
DB.query(INSERT_QUERY, **ad_data)
duration = (datetime.now() - start_time)
log1 = "loaded {} advertiser regional spend records for this week in {}".format(
total_rows, formattimedelta(duration))
log.info(log1)
info_to_slack("Google ads: " + log1)
if __name__ == "__main__":
# csvfn = os.path.join(os.path.dirname(__file__), '..', 'data/google-political-ads-transparency-bundle/google-political-ads-advertiser-weekly-spend.csv')
# with open(csvfn, 'r') as f:
# load_advertiser_weekly_spend_to_db(f)
from sys import argv
def get_bundle_from_zip(zip_fn):
return open(zip_fn, 'rb')
local_dest_for_bundle = os.path.join(os.path.dirname(__file__), '..',
'data')
# with get_current_bundle() as zip_file:
with get_bundle_from_zip(argv[1]) as zip_file:
explicit_bundle_date = datetime.date(
*map(int, argv[2].split("-"))) if len(argv) >= 3 else None
bundle_date = explicit_bundle_date or get_bundle_date(zip_file)
load_advertiser_regional_spend_to_db(
TextIOWrapper(BytesIO(
get_advertiser_regional_spend_csv(zip_file))), bundle_date)
def api_search():
json_response = json.load(TextIOWrapper(request.body))
print(json_response)
query = list(
filter(lambda x: len(list(x.keys())) > 0, json_response["query"]))
if len(query) == 0:
response.content_type = 'application/json'
return json.dumps([])
search_ranges = ["clause", "sentence", "paragraph", "verse", "phrase"]
search_range = json_response["search_range"]
if search_range not in search_ranges:
search_range = "clause"
search_range_filtered = get_filtered_search_range(
json_response["search_filter"] if "search_filter" in
json_response else [])
word_groups_to_exclude = []
word_group_with_match = [[] for i in range(len(query))]
found_words = []
for n in search_range_filtered:
inverted_search_done = False
regular_search_done = False
for q_index, q in enumerate(query):
query_inverted = "invert" in q
if (inverted_search_done
and query_inverted) or (regular_search_done
and not query_inverted):
continue
if test_node_with_query(n, q):
search_range_node = L.u(n, otype=search_range)[0]
word_group_with_match[q_index].append(search_range_node)
found_words.append({
"search_range_node": search_range_node,
"word_node": n
})
if query_inverted:
inverted_search_done = True
else:
regular_search_done = True
if regular_search_done and inverted_search_done:
break
words_groups_to_intersect = []
words_groups_to_filter = []
for q_index, q in enumerate(query):
if "invert" in q and q["invert"] == "t":
words_groups_to_filter += word_group_with_match[q_index]
else:
words_groups_to_intersect.append(word_group_with_match[q_index])
intersection_to_filter = list(
set.intersection(*map(set, words_groups_to_intersect)))
intersection = list(
filter(lambda x: x not in words_groups_to_filter,
intersection_to_filter))
print(str(len(intersection)) + " results")
# Truncate array if too long
truncated = False
if len(intersection) > 1000:
intersection = intersection[:500]
print("Abbreviating to just 500 elements")
truncated = True
retval = []
for r in intersection:
found_word_nodes = list(
map(lambda x: x["word_node"],
filter(lambda x: x["search_range_node"] == r, found_words)))
clause_text = get_highlighted_words_nodes_of_verse_range_from_node(
r, found_word_nodes)
retval.append({
"passage": passage_abbreviation(r),
"node": r,
"clause": clause_text,
})
# Grab parallel text
parallel_text = getPTextFromRefPairArray(
list(
map(
lambda x: (T.sectionFromNode(x["node"]),
T.sectionFromNode(x["node"], lastSlot=True)),
retval)))
if len(parallel_text) != len(retval):
print("how can we have different values?!?")
exit(1)
for i in range(len(parallel_text)):
retval[i]["english"] = parallel_text[i]
retval_sorted = sorted(retval, key=lambda r: sortKey(r["node"]))
response.content_type = 'application/json'
return json.dumps({
"truncated": truncated,
"search_results": retval_sorted
})
def log_subprocess_output(stdout):
for line in TextIOWrapper(stdout, encoding="utf-8"):
DESIGNER_LOG.info(line.rstrip('\n'))
print("Logging into Discord...")
bot.uptime = datetime.datetime.utcnow()
if bot.settings.login_credentials:
yield from bot.login(*bot.settings.login_credentials,
bot=not bot.settings.self_bot)
else:
print("No credentials available to login.")
raise RuntimeError()
yield from bot.connect()
if __name__ == '__main__':
sys.stdout = TextIOWrapper(sys.stdout.detach(),
encoding=sys.stdout.encoding,
errors="replace",
line_buffering=True)
bot = initialize()
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(bot))
except discord.LoginFailure:
bot.logger.error(traceback.format_exc())
if not bot.settings.no_prompt:
choice = input("Invalid login credentials. If they worked before "
"Discord might be having temporary technical "
"issues.\nIn this case, press enter and try again "
"later.\nOtherwise you can type 'reset' to reset "
"the current credentials and set them again the "
"next start.\n> ")
if choice.lower().strip() == "reset":
def importationProcess(request):
"""Importation des données des autres sites.
Certaines informations seront fournies par défaut.
Commande CODA: COM-INF 1
valeur : 0 XAF
Description des colonnes:
0: Site
1: Emplacement
2: Catégorie
3: Intitulé
4: Prix d'achat
5: Devise
6: Numéro de série
7: Fonctionnel
8: Usage
9: Date d'acquisition
10: Code d'inventaire
11: Code document
12: Numéro coda
13: Valeur Commande coda
14: Marque
15: Modèle
16: Description
17: Commentaire
"""
contexte = {}
if request.method == 'POST':
form = inventaireLegacyForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES['fichier']
if f.content_type == 'text/csv':
# les fichiers ici sont TOUS binaires d'où l'usage de io
#pour csv qui a besoin de fichiers textes
f = TextIOWrapper(f.file, encoding='UTF-8')
csvfile = csv.reader(f,delimiter=',')
premiere_ligne = True
ligne = 0
nbr_lignes_OK = 0
lignes_Not_OK = []
for colonne in csvfile:
ligne +=1
if premiere_ligne :
premiere_ligne = False
continue
#Implantation
try:
implantation0 = Implantation.objects.get(\
nom__icontains=colonne[0])
except Implantation.DoesNotExist:
implantation0 = Implantation(nom=colonne[0])
implantation0.save()
#Salle
try:
salle0 = implantation0.salle_set.get(\
nom__icontains=colonne[1])
except Salle.MultipleObjectsReturned:
salle0 = Salle.objects.filter(\
nom__icontains=colonne[1]).first()
except Salle.DoesNotExist:
salle0 = Salle(nom=colonne[1],site=implantation0)
salle0.save()
#Marque
if colonne[14] == '':
marque0,marque0_cree = Marque.objects.\
get_or_create(nom='INCONNU')
else:
try:
marque0 = Marque.objects.get(nom__icontains=\
colonne[14])
except Marque.MultipleObjectsReturned:
marque0 = Marque.objects.filter(\
nom__icontains=colonne[14]).first()
except Marque.DoesNotExist:
marque0 = Marque(nom=colonne[14])
marque0.save()
#Societe
soc0 = Societe.objects.get(nom='Divers')
#Produit
if colonne[15] == '':
produit0,produit0_cree = marque0.produit_set.\
get_or_create(modele='INCONNU')
else:
try:
produit0 = marque0.produit_set.get(\
modele__icontains=colonne[15])
except Produit.MultipleObjectsReturned :
produit0 = marque0.produit_set.filter(\
modele__icontains=colonne[15]).first()
except Produit.DoesNotExist:
produit0 = Produit(modele=colonne[15],
constructeur=marque0)
produit0.save()
try:
devise0 = Devise.objects.get(\
identifiant=colonne[5])
except Devise.DoesNotExist:
devise0 = Devise.objects.get(identifiant='XAF')
descr0 = ''
comm0 = ''
p_achat0 = 0
usage0 = 'personnel'
if colonne[4] != '':
p_achat0 = int(colonne[4])
if colonne[6] != '':
num_serie0 = colonne[6]
else:
num_serie0 = ''
if colonne[7] == 'True':
fonctionnel0 = True
else:
fonctionnel0 = False
if colonne[8] != '':
for u in Piece.USAGE:
if colonne[8] == u[0]:
usage0 = colonne[8]
break
if colonne[9] == '':
date_acquisition0 = date(year=date.today().year-2,
month=date.today().month,
day=date.today().day
)
else:
if len(colonne[9]) == 4:
date_acquisition0 = date(
year=int(colonne[9]),
month=7,day=14
)
else:
#format colonne Buja 06-12-2009
moment = colonne[9]
date_acquisition0 = date(
year=int(moment[:4]),
month=int(moment[5:7]),
day=int(moment[8:]),
)
if colonne[10] == '':
code_inventaire0 = code_aleatoire()
else:
code_inventaire0 = colonne[10]
if colonne[16] != '':
descr0 = colonne[16]
if colonne[17] != '':
comm0 = colonne[17]
#Commande
if colonne[11] == '':
commande_coda0 = Commande.objects.get(numero=89)
else:
try:
commande_coda0 = Commande.objects.get(\
section=colonne[11],
numero=int(colonne[12]))
except Commande.DoesNotExist:
if colonne[5] == '':
devise0 = Devise.objects.get(\
identifiant='EUR')
else:
devise0 = Devise.objects.get(\
identifiant=colonne[5])
try:
commande_coda0 = Commande(\
valeur=p_achat0,
#section=colonne[11],
numero=int(colonne[12]),
devise=devise0,
livree_par=soc0,
)
commande_coda0.save()
except:
comm0 = 'souci avec la comm coda'
commande_coda0 = \
Commande.objects.get(numero=89)
piece = Piece(intitule = colonne[3],
prix_achat = p_achat0,
devise = devise0,
num_serie = num_serie0,
fonctionnel = fonctionnel0,
usage = usage0,
date_acquisition = date_acquisition0,
#date_acquisition = date(year=date.today().year-2,
# month=date.today().month,
# day=date.today().day),
code_inventaire = code_inventaire0,
description = descr0,
commentaire = comm0,
emplacement = salle0,
#on rappelle qu'on se sert par défaut de
#la COM-INF 1
#Pour la démo, j'ai fixé à 89
#TODO à améliorer
commande_coda = commande_coda0,
# pk=commande_coda0.id),
modele = produit0,
categorie = Categorie.objects.get(\
nom__icontains=colonne[2]),
)
piece.save()
nbr_lignes_OK += 1
#else:
# cette pièce a déjà été inventoriée
# lignes_Not_OK.append(ligne)
contexte = {'lignes_Not_OK': lignes_Not_OK,
'nbr_lignes_OK': nbr_lignes_OK,
'total_lignes': ligne,
'f': f,
}
return render(request,
'inventaire2/importation-csv-report.html',
contexte, status=302)
else :
form = inventaireLegacyForm()
contexte = {'message': "votre fichier n'est pas au \
format CSV",
'form': form,
'f':f,
'F':dir(f),
}
return render(request, 'inventaire2/importation-csv.html',
contexte,status=302)
else:
form = inventaireLegacyForm()
contexte['message'] = "le formulaire n'est pas valide"
return HttpResponseRedirect(reverse('importation',
kwargs = contexte))
#return render(request,
#'inventaire2/importation-csv.html',
#contexte,
#)
else:
form = inventaireLegacyForm()
contexte = {'message': 'Veuillez recommencer svp',
'form': form }
return render(request,'inventaire2/importation-csv.html',
contexte)
def load_sql(file):
return ''.join(x for x in TextIOWrapper(resource_stream('CGRdb.sql', file))
if not x.startswith(('#', '/*', '*/', '\n'))).replace('$', '$$')
def post(self, request, *args, **kwargs):
project = get_object_or_404(Project, pk=kwargs.get('project_id'))
try:
if 'key_name' in request.POST:
key_name = request.POST['key_name']
if project.is_type_of(Project.SEQUENCE_LABELING):
Document.objects.bulk_create([
Document(text=line.strip(), project=project)
for line in form_data
])
else:
# get a handle on s3
s3 = boto3.resource('s3')
# get a handle on the bucket that holds your file
bucket = s3.Bucket('go-mmt-data-science')
# get a handle on the object you want (i.e. your file)
obj = bucket.Object(
key='chat_bot/tool_data/{0}'.format(key_name))
# get the object
response = obj.get()
# form_data = response['Body'].read().splitlines(True)
df = pd.read_csv(response['Body'])
print(df.shape)
# create dataset from S3 file
doc_objs = []
label_objs = []
doc_ids = {d.id for d in Document.objects.all()}
for (_, line) in df.iterrows():
if (line[0].strip() not in doc_ids):
doc_objs.append(
Document(id=line[0].strip(),
create_date=pd.to_datetime(
line[1].strip()),
source=line[2].strip(),
text=line[3].strip(),
project=project))
for (idx, label) in enumerate(line[4:]):
label_objs.append(
Label(text=label.strip(),
shortcut=chr(ord('a') + idx),
project=project,
documents_id=line[0].strip()))
Document.objects.bulk_create(doc_objs)
Label.objects.bulk_create(label_objs)
# for (_, line) in df.iterrows():
# doc = Document(id=line[0].strip(), text=line[1].strip(), project=project)
# doc.save()
# print([doc.doc_labels.create(text=label.strip(), shortcut=chr(ord('a') + idx), project=project)
# for
# (idx, label) in enumerate(line[2:])])
else:
form_data = TextIOWrapper(request.FILES['csv_file'].file,
encoding='utf-8')
reader = csv.reader(form_data)
next(reader, None) # skip the headers
doc_objs = []
label_objs = []
doc_ids = {d.id for d in Document.objects.all()}
for line in reader:
if (line[0].strip() not in doc_ids):
doc_objs.append(
Document(id=line[0].strip(),
create_date=pd.to_datetime(
line[1].strip()),
source=line[2].strip(),
text=line[3].strip(),
project=project))
for (idx, label) in enumerate(line[4:]):
label_objs.append(
Label(text=label.strip(),
shortcut=chr(ord('a') + idx),
project=project,
documents_id=line[0].strip()))
Document.objects.bulk_create(doc_objs)
Label.objects.bulk_create(label_objs)
# for line in reader:
# doc = Document(id=line[0].strip(), text=line[1].strip(), project=project)
# doc.save()
# print([doc.doc_labels.create(text=label.strip(), shortcut=chr(ord('a') + idx), project=project) for
# (idx, label) in enumerate(line[2:])])
return HttpResponseRedirect(reverse('dataset', args=[project.id]))
except Exception as E:
print(E)
return HttpResponseRedirect(reverse('upload', args=[project.id]))
def test_main(capsys, tmpdir):
base_args = [
"-sp",
str(tmpdir),
"--virtual-env",
str(tmpdir),
"--src-path",
str(tmpdir),
]
tmpdir.mkdir(".git")
# If nothing is passed in the quick guide is returned without erroring
main.main([])
out, error = capsys.readouterr()
assert main.QUICK_GUIDE in out
assert not error
# If no files are passed in but arguments are the quick guide is returned, alongside an error.
with pytest.raises(SystemExit):
main.main(base_args)
out, error = capsys.readouterr()
assert main.QUICK_GUIDE in out
# Unless the config is requested, in which case it will be returned alone as JSON
main.main(base_args + ["--show-config"])
out, error = capsys.readouterr()
returned_config = json.loads(out)
assert returned_config
assert returned_config["virtual_env"] == str(tmpdir)
# This should work even if settings path is not provided
main.main(base_args[2:] + ["--show-config"])
out, error = capsys.readouterr()
assert json.loads(out)["virtual_env"] == str(tmpdir)
# This should raise an error if an invalid settings path is provided
with pytest.raises(InvalidSettingsPath):
main.main(
base_args[2:]
+ ["--show-config"]
+ ["--settings-path", "/random-root-folder-that-cant-exist-right?"]
)
# Should be able to set settings path to a file
config_file = tmpdir.join(".isort.cfg")
config_file.write(
"""
[settings]
profile=hug
verbose=true
"""
)
config_args = ["--settings-path", str(config_file)]
main.main(
config_args
+ ["--virtual-env", "/random-root-folder-that-cant-exist-right?"]
+ ["--show-config"]
)
out, error = capsys.readouterr()
assert json.loads(out)["profile"] == "hug"
# Should be able to stream in content to sort
input_content = TextIOWrapper(
BytesIO(
b"""
import b
import a
"""
)
)
main.main(config_args + ["-"], stdin=input_content)
out, error = capsys.readouterr()
assert (
out
== f"""
else-type place_module for b returned {DEFAULT_CONFIG.default_section}
else-type place_module for a returned {DEFAULT_CONFIG.default_section}
import a
import b
"""
)
# Should be able to stream diff
input_content = TextIOWrapper(
BytesIO(
b"""
import b
import a
"""
)
)
main.main(config_args + ["-", "--diff"], stdin=input_content)
out, error = capsys.readouterr()
assert not error
assert "+" in out
assert "-" in out
assert "import a" in out
assert "import b" in out
# check should work with stdin
input_content_check = TextIOWrapper(
BytesIO(
b"""
import b
import a
"""
)
)
with pytest.raises(SystemExit):
main.main(config_args + ["-", "--check-only"], stdin=input_content_check)
out, error = capsys.readouterr()
assert error == "ERROR: Imports are incorrectly sorted and/or formatted.\n"
# Should be able to run with just a file
python_file = tmpdir.join("has_imports.py")
python_file.write(
"""
import b
import a
"""
)
main.main([str(python_file), "--filter-files", "--verbose"])
assert python_file.read().lstrip() == "import a\nimport b\n"
# Add a file to skip
should_skip = tmpdir.join("should_skip.py")
should_skip.write("import nothing")
main.main(
[
str(python_file),
str(should_skip),
"--filter-files",
"--verbose",
"--skip",
str(should_skip),
]
)
# Should raise a system exit if check only, with broken file
python_file.write(
"""
import b
import a
"""
)
with pytest.raises(SystemExit):
main.main(
[
str(python_file),
str(should_skip),
"--filter-files",
"--verbose",
"--check-only",
"--skip",
str(should_skip),
]
)
# Should have same behavior if full directory is skipped
with pytest.raises(SystemExit):
main.main(
[str(tmpdir), "--filter-files", "--verbose", "--check-only", "--skip", str(should_skip)]
)
# Nested files should be skipped without needing --filter-files
nested_file = tmpdir.mkdir("nested_dir").join("skip.py")
nested_file.write("import b;import a")
python_file.write(
"""
import a
import b
"""
)
main.main([str(tmpdir), "--extend-skip", "skip.py", "--check"])
# without filter options passed in should successfully sort files
main.main([str(python_file), str(should_skip), "--verbose", "--atomic"])
# Should raise a system exit if all passed path is broken
with pytest.raises(SystemExit):
main.main(["not-exist", "--check-only"])
# Should not raise system exit if any of passed path is not broken
main.main([str(python_file), "not-exist", "--verbose", "--check-only"])
out, error = capsys.readouterr()
assert "Broken" in out
# warnings should be displayed if old flags are used
with pytest.warns(UserWarning):
main.main([str(python_file), "--recursive", "-fss"])
# warnings should be displayed when streaming input is provided with old flags as well
with pytest.warns(UserWarning):
main.main(["-sp", str(config_file), "-"], stdin=input_content)
def _wrapStream(self, stream):
return TextIOWrapper(stream,
encoding=self.encoding,
write_through=True)
def save(self, commit=True):
csv_file = self.cleaned_data['csv_file']
clear_existing_tags = self.cleaned_data['clear_existing_tags']
f = TextIOWrapper(
csv_file.file,
encoding=csv_file.charset if csv_file.charset else 'utf-8')
records = csv.DictReader(f)
import_errors = False
import_success = False
count = 0
count_imported = 0
count_notfound = 0
if records:
# the first row is the tags
for row in records:
count += 1
topic = row.get('topic')
if not topic:
topic = row.get('__topic')
entities = Entity.objects.filter(topic=topic)
if entities:
for entity in entities:
if clear_existing_tags:
entity.remove_all_tags(commit=False)
for tag in row.keys():
if tag != 'topic' and not tag.startswith('__'):
value = row.get(tag)
# check tag Kind
if value:
try:
tag_object = Tag.objects.get(tag=tag)
except Tag.DoesNotExist:
logging.error(
"Cannot get tag {}".format(tag))
else:
if tag_object.kind == 'Marker':
entity.add_tag(tag,
value=None,
commit=False)
else:
entity.add_tag(tag,
value=value,
commit=False)
entity.save()
count_imported += 1
else:
count_notfound += 1
else:
import_errors = 'Cannot parse source file'
if count:
if count_imported:
t = "row" if count_imported == 1 else "rows"
import_success = "Imported {count} {t} successfully. ".format(
count=count_imported, t=t)
else:
import_success = ""
if count_notfound:
t = "row" if count_notfound == 1 else "rows"
import_success += "{count} {t} from your CSV could not be found.".format(
count=count_notfound, t=t)
else:
import_errors = 'There is no any data row in the source file'
result = {}
if import_errors:
result['import_errors'] = import_errors
elif import_success:
result['import_success'] = import_success
return result
def handle_files(request):
# Lê o arquivo com os dados dos alunos e gera um dicionário do Python com eles.
f = TextIOWrapper(request.FILES['file'].file, encoding=request.encoding)
reader = csv.DictReader(f)
# Lê cada linha do dicionário contendo os dados dos alunos. O que corresponde aos dados de um
# aluno em específico.
for row in reader:
# Procura por valores vazios na linha atual do dicionário de alunos, e
# altera esses valores de NULL para o valor do Python Nome.
for r in row.keys():
if row[r] == 'NULL':
row[r] = None
# Altera a formatação da data de nascimento do aluno em questão.
data_nascimento_aluno = is_none(row['DT_NASCIMENTO'])
if data_nascimento_aluno:
# Altera o formato da data para ficar igual ao usado pelo banco de dados.
try:
data_nascimento_aluno = data_nascimento_aluno.split("/")
data_nascimento_aluno = data_nascimento_aluno[2] + "-" + data_nascimento_aluno[1] + "-" + \
data_nascimento_aluno[0]
# Caso não consiga por não ser um dado válido, apague-o com o valor None e emita uma mensagem de erro.
except Exception as e:
data_nascimento_aluno = None
print("Erro ao parsear a Data de Nascimento do cpf: " +
is_none(row['CPF']))
print(e)
# Verifica se já existe no banco os dados pessoais de determinado aluno com o cpf indicado e guarda o id
# desse aluno. Caso não exista, o id recebe o valor None.
try:
objPessoa = Pessoa.objects.get(cpf=is_none(row['CPF']))
id = objPessoa.pk
except:
id = None
# Cria um objeto (modelo/registro) Pessoa (que conteria os dados pessoais do aluno).
obj = Pessoa(
cpf=is_none(row['CPF']),
nome=is_none(row['NOME_ALUNO']),
dt_nasc=data_nascimento_aluno,
tipo_endereco=is_none(row['TIPO_LOGRADOURO']),
endereco=is_none(row['LOGRADOURO']),
numero=is_none(row['NUMERO']),
complemento=is_none(row['COMPLEMENTO']),
bairro=is_none(row['BAIRRO']),
cep=is_none(row['CEP']),
# distrito=is_none(row['DISTRITO']),
cidade=is_none(row['MUNICIPIO']),
estado=is_none(row['UF']),
pais=is_none(row['PAIS']),
email=is_none(row['E_MAIL']),
# ddi_residencial=is_none(row['DDI_RESIDENCIAL']),
# ddd_residencial=is_none(row['DDD_RESIDENCIAL']),
fone_residencial01=is_none(row['FONE_RESIDENCIAL']),
fone_residencial02=None,
# ddi_comercial=is_none(row['DDI_COMERCIAL']),
# ddd_comercial=is_none(row['DDD_COMERCIAL']),
fone_comercial01=is_none(row['FONE_COMERCIAL']),
fone_comercial02=None,
# ddi_celular=is_none(row['DDI_CELULAR']),
# ddd_celular=is_none(row['DDD_CELULAR']),
fone_celular01=is_none(row['FONE_CELULAR']),
fone_celular02=None)
# Se o valor de id não for None (os dados pessoais do aluno já existem no banco de dados), acrescenta o id no
# objeto e atualiza-o no banco de dados. Exibe um erro se não conseguir e pula esse registro do
# arquivo carregado.
if id:
obj.id = id
try:
obj.save()
except Exception as err:
print(err)
continue
# Verifica se o curso existe no banco de dados, e cria-o se não existir.
try:
objCurso = Curso.objects.get(cod_curso=is_none(row['COD_CURSO']))
except:
objCurso = Curso.objects.create(cod_curso=is_none(
row['COD_CURSO']),
nome_curso=is_none(row['CURSO']))
# Verifica se a unidade (ou campus) existe no banco de dados, e cria-a e salva-a se não existir.
try:
objCampus = Campus.objects.get(
nome_do_campus=is_none(row['UNIDADE']))
except:
objCampus = Campus.objects.create(
nome_do_campus=is_none(row['UNIDADE']))
# Salva os dados do campus.
try:
objCampus.save()
except Exception as err:
print(err)
# Verifica se a matrícula (o aluno) já existe no banco de dados através do uso da chave primária, e
# guarda-o se existir. Caso não exista, configura o objeto para None e não salva no banco de dado
# essas informações.
try:
objAluno = Aluno.objects.filter(id_aluno__pk=obj.pk)
except:
objAluno = None
# Existindo algum aluno (ou vários alunos) com determinada matrícula, separa a chave primária de cada curso
# numa lista, e ??? Se tiver a chave do curso encontrado no registro atual do aluno, pula esse registro.
if objAluno:
lis = list()
for valueobj in objAluno:
lis.append(valueobj.nome_curso.pk)
if objCurso.pk in lis:
continue
# Cria o objeto aluno e salva-o no banco de dados.
obj_ps = Aluno(matricula=is_none(row['MATRICULA']),
situacao=is_none(row['SITUACAO']),
id_curso=objCurso,
id_aluno=obj)
obj_ps.save()
return True
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool( processes=BibleOrgSysGlobals.maxProcesses ) as pool: # start worker processes
results = pool.map( testDB, parameters ) # have the pool do our loads
assert len(results) == len(parameters) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate( sorted( foundFolders ) ):
if BibleOrgSysGlobals.verbosityLevel > 1: print( "\nDrupalBible D{}/ Trying {}".format( j+1, someFolder ) )
#myTestFolder = os.path.join( testFolder, someFolder+'/' )
testDB( someFolder )
# end of demo
if __name__ == '__main__':
multiprocessing.freeze_support() # Multiprocessing support for frozen Windows executables
import sys
if 'win' in sys.platform: # Convert stdout so we don't get zillions of UnicodeEncodeErrors
from io import TextIOWrapper
sys.stdout = TextIOWrapper( sys.stdout.detach(), sys.stdout.encoding, 'namereplace' if sys.version_info >= (3,5) else 'backslashreplace' )
# Configure basic Bible Organisational System (BOS) set-up
# Configure basic set-up
parser = BibleOrgSysGlobals.setup( ProgName, ProgVersion )
BibleOrgSysGlobals.addStandardOptionsAndProcess( parser, exportAvailable=True )
demo()
BibleOrgSysGlobals.closedown( ProgName, ProgVersion )
# end of DrupalBible.py
data.append([
'**' + p['description'] + '**',
', '.join(['``' + str(f) + '``' for f in p['fieldnames']]),
', '.join([{
'orbit': '`~sbpy.data.Orbit`',
'ephem': '`~sbpy.data.Ephem`',
'obs': '`~sbpy.data.Obs`',
'phys': '`~sbpy.data.Phys`'
}[m.replace(',', '')] for m in p['provenance']]),
str(p['dimension'])
])
data = Table(array(data),
names=('Description', 'Field Names', 'Provenance', 'Dimension'))
# redirect stdout
sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding)
# ascii.write will write to stdout
datarst = ascii.write(data, format='rst')
# read formatted table data
sys.stdout.seek(0)
rsttab = sys.stdout.read()
# add formatted table data to out
out += rsttab
# write fieldnames.rst
with open('sbpy/data/fieldnames.rst', 'w') as outf:
outf.write(out)
def createGeoJson(geoms, output=None, srs=4326, topo=False, fill=''):
"""Convert a set of geometries to a geoJSON object"""
if(srs):
srs = SRS.loadSRS(srs)
# arrange geom, index, and data
if isinstance(geoms, ogr.Geometry): # geoms is a single geometry
finalGeoms = [geoms, ]
data = None
index = [0, ]
elif isinstance(geoms, pd.Series):
index = geoms.index
finalGeoms = geoms.values
data = None
elif isinstance(geoms, pd.DataFrame):
index = geoms.index
finalGeoms = geoms.geom.values
data = geoms.loc[:, geoms.columns != 'geom']
data["_index"] = index
else:
finalGeoms = list(geoms)
data = None
index = list(range(len(finalGeoms)))
if len(finalGeoms) == 0:
raise GeoKitVectorError("Empty geometry list given")
# Transform?
if not srs is None:
finalGeoms = GEOM.transform(finalGeoms, toSRS=srs)
# Make JSON object
from io import BytesIO
if not output is None and not isinstance(output, str):
if not output.writable():
raise GeoKitVectorError("Output object is not writable")
if topo:
fo = BytesIO()
else:
fo = output
elif isinstance(output, str) and not topo:
fo = open(output, "wb")
else:
fo = BytesIO()
fo.write(
bytes('{"type":"FeatureCollection","features":[', encoding='utf-8'))
for j, i, g in zip(range(len(index)), index, finalGeoms):
fo.write(bytes('%s{"type":"Feature",' %
("" if j == 0 else ","), encoding='utf-8'))
if data is None:
fo.write(
bytes('"properties":{"_index":%s},' % str(i), encoding='utf-8'))
else:
fo.write(bytes('"properties":%s,' %
data.loc[i].fillna(fill).to_json(), encoding='utf-8'))
fo.write(bytes('"geometry":%s}' % g.ExportToJson(), encoding='utf-8'))
#fo.write(bytes('"geometry": {"type": "Point","coordinates": [125.6, 10.1] }}', encoding='utf-8'))
fo.write(bytes("]}", encoding='utf-8'))
fo.flush()
# Put in the right format
if topo:
from topojson import conversion
from io import TextIOWrapper
fo.seek(0)
topo = conversion.convert(TextIOWrapper(
fo), object_name="primary") # automatically closes fo
topo = str(topo).replace("'", '"')
# Done!
if output is None:
if topo:
return topo
else:
fo.seek(0)
geojson = fo.read()
fo.close()
return geojson.decode('utf-8')
elif isinstance(output, str):
if topo:
with open(output, "w") as fo:
fo.write(topo)
else:
pass # we already wrote to the file!
return output
else:
if topo:
output.write(bytes(topo, encoding='utf-8'))
else:
pass # We already wrote to the file!
return None
def auto_input(request):
"""
Allows users to upload a file of variants to classify.
"""
PANEL_OPTIONS = [(str(panel.pk), panel)
for panel in Panel.objects.all().order_by('panel')]
# Inititate the upload form and context dict to pass to template
form = VariantFileUploadForm(options=PANEL_OPTIONS)
context = {
'form': form,
'error': None,
'warn': None,
'success': None,
'params': None
}
if request.POST:
form = VariantFileUploadForm(request.POST,
request.FILES,
options=PANEL_OPTIONS)
if form.is_valid():
# get panel
analysis_performed_pk = form.cleaned_data['panel_applied'].lower()
panel_obj = get_object_or_404(Panel, panel=analysis_performed_pk)
# get affected with
affected_with = form.cleaned_data['affected_with']
# process tsv file
raw_file = request.FILES['variant_file']
utf_file = TextIOWrapper(raw_file, encoding='utf-8')
df, meta_dict = load_worksheet(utf_file)
# Get key information from the dataframe
# We have checked that there is only one sample/worksheet in the df
unique_variants = df['Variant'].unique()
worksheet_id = df['WorklistId'].unique()[0]
sample_id = df['#SampleId'].unique()[0]
# get reference genome
if 'Reference' in df:
genome = df['Reference'].unique()[0]
else:
genome = 'GRCh37'
# create dict of links between variant and genotype
variant_genotype_dict = {}
for row in df.itertuples():
variant_genotype_dict[row.Variant] = row.Genotype
# add worksheet
worksheet_obj, created = Worklist.objects.get_or_create(
name=worksheet_id)
# add sample
try:
sample_obj = Sample.objects.get(name=worksheet_id + '-' +
sample_id + '-' +
analysis_performed_pk)
# throw error if the sample has been uploaded before with the same panel (wont throw error if its a different panel)
context['error'] = [
f'ERROR: {sample_obj.name} has already been uploaded with the {analysis_performed_pk} panel.'
]
return render(request, 'acmg_db/auto_input.html', context)
except Sample.DoesNotExist:
sample_obj = Sample.objects.create(
name=worksheet_id + '-' + sample_id + '-' +
analysis_performed_pk,
sample_name_only=sample_id,
worklist=worksheet_obj,
affected_with=affected_with,
analysis_performed=panel_obj,
analysis_complete=False,
other_changes='',
genome=genome)
sample_obj.save()
# Get VEP annotations
# Set dictionary depending on Reference genome input from form
if genome == "GRCh37":
vep_info_dict = {
'reference_genome': settings.REFERENCE_GENOME_37,
'vep_cache': settings.VEP_CACHE_37,
'temp_dir': settings.VEP_TEMP_DIR,
'assembly': settings.ASSEMBLY_37,
'version': settings.VEP_VERSION_37
}
elif genome == "GRCh38":
vep_info_dict = {
'reference_genome': settings.REFERENCE_GENOME_38,
'vep_cache': settings.VEP_CACHE_38,
'temp_dir': settings.VEP_TEMP_DIR,
'assembly': settings.ASSEMBLY_38,
'version': settings.VEP_VERSION_38
}
variant_annotations = get_vep_info_local(unique_variants,
vep_info_dict, sample_id)
# Loop through each variant and add to the database
for variant in variant_annotations:
var = variant[1]
variant_data = process_variant_input(var, genome)
variant_hash = variant_data[0]
chromosome = variant_data[1]
position = variant_data[2]
ref = variant_data[3]
alt = variant_data[4]
variant_obj, created = Variant.objects.get_or_create(
variant_hash=variant_hash,
chromosome=chromosome,
position=position,
ref=ref,
alt=alt,
genome=genome)
if 'transcript_consequences' in variant[0]:
consequences = variant[0]['transcript_consequences']
elif 'intergenic_consequences' in variant[0]:
consequences = variant[0]['intergenic_consequences']
else:
raise Exception(
f'Could not get the consequences for variant {variant}'
)
selected = None
# Loop through each consequence/transcript
for consequence in consequences:
if 'transcript_id' in consequence:
transcript_id = consequence['transcript_id']
else:
transcript_id = 'None'
transcript_hgvsc = consequence.get('hgvsc')
transcript_hgvsp = consequence.get('hgvsp')
gene_symbol = consequence.get('gene_symbol', 'None')
exon = consequence.get('exon', 'NA')
impact = consequence.get('consequence_terms')
impact = '|'.join(impact)
gene_obj, created = Gene.objects.get_or_create(
name=gene_symbol)
transcript_obj, created = Transcript.objects.get_or_create(
name=transcript_id, gene=gene_obj)
transcript_variant_obj, created = TranscriptVariant.objects.get_or_create(
variant=variant_obj,
transcript=transcript_obj,
hgvs_c=transcript_hgvsc,
hgvs_p=transcript_hgvsp,
exon=exon,
consequence=impact)
# only add the vep version if its a new transcript, otherwise there will be duplicates for each vep version
if genome == "GRCh37":
vep_version = settings.VEP_VERSION_37
elif genome == "GRCh38":
vep_version = settings.VEP_VERSION_38
if created:
transcript_variant_obj.vep_version = vep_version
transcript_variant_obj.save()
# Find the transcript that VEP has picked
if 'pick' in consequence:
selected = transcript_variant_obj
# process the genotype
genotype = variant_genotype_dict[
f'{variant_obj.chromosome}:{variant_obj.position}{variant_obj.ref}>{variant_obj.alt}']
# for FH TSVs in which the genotype field is different e.g. A/G rather than HET
if '/' in genotype:
alt = variant_obj.alt
genotype = genotype.split('/')
if genotype.count(alt) == 1:
genotype = 'HET'
elif genotype.count(alt) == 2:
genotype = 'HOM'
else:
# something else set to None
genotype = None
if genotype == 'HET':
genotype = 1
elif genotype == 'HOM' or genotype == 'HOM_ALT':
genotype = 2
else:
genotype = None
new_classification_obj = Classification.objects.create(
variant=variant_obj,
sample=sample_obj,
creation_date=timezone.now(),
user_creator=request.user,
status='0',
is_trio_de_novo=False,
first_final_class='7',
second_final_class='7',
selected_transcript_variant=selected,
genotype=genotype,
guideline_version=guideline_version,
vep_version=vep_version)
new_classification_obj.save()
success = [
'Worksheet {} - Sample {} - {} panel - Upload completed '.
format(worksheet_id, sample_id, analysis_performed_pk)
]
params = '?worksheet={}&sample={}&panel={}'.format(
worksheet_id, sample_id, analysis_performed_pk)
context = {'form': form, 'success': success, 'params': params}
return render(request, 'acmg_db/auto_input.html', context)
def api_book_chapter():
json_response = json.load(TextIOWrapper(request.body))
print(json_response)
reference = json_response["reference"]
book = generous_name(reference["book"])
chapter = int(
reference["chapter"]) # This needs to be a string for the if...
book_chapter_node = T.nodeFromSection((book, chapter))
if "display_by" not in json_response:
json_response["display_by"] = "verse"
display_setting_options = ["verse", "clause"]
display_setting = json_response["display_by"] if json_response[
"display_by"] in display_setting_options else "verse"
highlights = {}
if "highlights" in json_response:
highlights = json_response["highlights"]
chapter_data = []
if display_setting == "clause":
for c in L.d(book_chapter_node, otype='clause_atom'):
clause_atom_words = []
accent_unit_chunk = []
previous_trailer = " "
for w in L.d(c, otype='word'):
highlightMatches = [
k for k, c in highlights.items()
if test_node_with_query(w, c)
]
if previous_trailer not in {'', '־'}:
if len(accent_unit_chunk) > 0:
clause_atom_words.append(accent_unit_chunk)
accent_unit_chunk = []
accent_unit_chunk.append({
"wid": w,
"bit": F.g_word_utf8.v(w),
"trailer": F.trailer_utf8.v(w),
"highlights": highlightMatches,
})
previous_trailer = F.trailer_utf8.v(w)
if len(accent_unit_chunk) > 0:
clause_atom_words.append(accent_unit_chunk)
chapter_data.append({
"verse": T.sectionFromNode(c)[2],
"tab": F.tab.v(c),
"type": F.typ.v(c),
"clause_words": clause_atom_words,
})
elif display_setting == "verse":
for v in L.d(book_chapter_node, otype='verse'):
verse_words = []
accent_unit_chunk = []
previous_trailer = " "
for w in L.d(v, otype='word'):
highlightMatches = [
k for k, v in highlights.items()
if test_node_with_query(w, v)
]
if previous_trailer not in {'', '־'}:
if len(accent_unit_chunk) > 0:
verse_words.append(accent_unit_chunk)
accent_unit_chunk = []
accent_unit_chunk.append({
"wid": w,
"bit": F.g_word_utf8.v(w),
"trailer": F.trailer_utf8.v(w),
"highlights": highlightMatches,
})
previous_trailer = F.trailer_utf8.v(w)
if len(accent_unit_chunk) > 0:
verse_words.append(accent_unit_chunk)
chapter_data.append({
"verse": F.verse.v(v),
"verse_words": verse_words
})
response.content_type = 'application/json'
ret = {
"reference": {
"book": book,
"chapter": chapter
},
"display_by": display_setting,
"chapter_data": chapter_data
}
return json.dumps(ret)
import csv
from io import TextIOWrapper
from zipfile import ZipFile
serList = [] #for tracking serial numbers
with open("harddrive_begin_status", "w", newline="") as file:
writer = csv.writer(file)
writer.writerow(["start_date", "serial_number", "model"])
with ZipFile("data_Q4_2019.zip") as zf:
fileList = ZipFile.namelist(zf)
for i in fileList:
with zf.open(i, "r") as infile:
reader = csv.reader(TextIOWrapper(infile))
for row in reader:
if row[2] == "ST12000NM0008" or \
row[2] == "TOSHIBA MG07ACA14TA":
if row[1] not in serList:
serList.append(row[1])
writer.writerow([row[0], row[1], row[2]])
rest_files_list = ["data_Q1_2020.zip", "data_Q2_2020.zip", \
"data_Q3_2020.zip"]
with open("harddrive_begin_status", "a", newline="") as file:
for j in rest_files_list:
writer = csv.writer(file)
with ZipFile(j) as zf:
fileList = ZipFile.namelist(zf)
for i in fileList:
def _read_stdin():
# https://mail.python.org/pipermail/python-list/2012-November/634424.html
sys.stdin = TextIOWrapper(sys.stdin.detach(), encoding="utf-8")
return sys.stdin.read()
def _print_figure(self,
outfile,
format,
*,
dpi,
dsc_comments,
orientation,
papertype,
bbox_inches_restore=None):
"""
Render the figure to a filesystem path or a file-like object.
Parameters are as for `.print_figure`, except that *dsc_comments* is a
all string containing Document Structuring Convention comments,
generated from the *metadata* parameter to `.print_figure`.
"""
is_eps = format == 'eps'
if isinstance(outfile, (str, os.PathLike)):
outfile = os.fspath(outfile)
passed_in_file_object = False
elif is_writable_file_like(outfile):
passed_in_file_object = True
else:
raise ValueError("outfile must be a path or a file-like object")
# find the appropriate papertype
width, height = self.figure.get_size_inches()
if papertype == 'auto':
papertype = _get_papertype(
*orientation.swap_if_landscape((width, height)))
paper_width, paper_height = orientation.swap_if_landscape(
papersize[papertype])
if mpl.rcParams['ps.usedistiller']:
# distillers improperly clip eps files if pagesize is too small
if width > paper_width or height > paper_height:
papertype = _get_papertype(
*orientation.swap_if_landscape((width, height)))
paper_width, paper_height = orientation.swap_if_landscape(
papersize[papertype])
# center the figure on the paper
xo = 72 * 0.5 * (paper_width - width)
yo = 72 * 0.5 * (paper_height - height)
llx = xo
lly = yo
urx = llx + self.figure.bbox.width
ury = lly + self.figure.bbox.height
rotation = 0
if orientation is _Orientation.landscape:
llx, lly, urx, ury = lly, llx, ury, urx
xo, yo = 72 * paper_height - yo, xo
rotation = 90
bbox = (llx, lly, urx, ury)
self._pswriter = StringIO()
# mixed mode rendering
ps_renderer = RendererPS(width, height, self._pswriter, imagedpi=dpi)
renderer = MixedModeRenderer(self.figure,
width,
height,
dpi,
ps_renderer,
bbox_inches_restore=bbox_inches_restore)
self.figure.draw(renderer)
def print_figure_impl(fh):
# write the PostScript headers
if is_eps:
print("%!PS-Adobe-3.0 EPSF-3.0", file=fh)
else:
print(
f"%!PS-Adobe-3.0\n"
f"%%DocumentPaperSizes: {papertype}\n"
f"%%Pages: 1\n",
end="",
file=fh)
print(
f"{dsc_comments}\n"
f"%%Orientation: {orientation.name}\n"
f"{get_bbox_header(bbox)[0]}\n"
f"%%EndComments\n",
end="",
file=fh)
Ndict = len(psDefs)
print("%%BeginProlog", file=fh)
if not mpl.rcParams['ps.useafm']:
Ndict += len(ps_renderer._character_tracker.used)
print("/mpldict %d dict def" % Ndict, file=fh)
print("mpldict begin", file=fh)
print("\n".join(psDefs), file=fh)
if not mpl.rcParams['ps.useafm']:
for font_path, chars \
in ps_renderer._character_tracker.used.items():
if not chars:
continue
font = get_font(font_path)
glyph_ids = [font.get_char_index(c) for c in chars]
fonttype = mpl.rcParams['ps.fonttype']
# Can't use more than 255 chars from a single Type 3 font.
if len(glyph_ids) > 255:
fonttype = 42
fh.flush()
if fonttype == 3:
fh.write(_font_to_ps_type3(font_path, glyph_ids))
else:
try:
convert_ttf_to_ps(os.fsencode(font_path), fh,
fonttype, glyph_ids)
except RuntimeError:
_log.warning(
"The PostScript backend does not currently "
"support the selected font.")
raise
print("end", file=fh)
print("%%EndProlog", file=fh)
if not is_eps:
print("%%Page: 1 1", file=fh)
print("mpldict begin", file=fh)
print("%s translate" % _nums_to_str(xo, yo), file=fh)
if rotation:
print("%d rotate" % rotation, file=fh)
print("%s clipbox" % _nums_to_str(width * 72, height * 72, 0, 0),
file=fh)
# write the figure
print(self._pswriter.getvalue(), file=fh)
# write the trailer
print("end", file=fh)
print("showpage", file=fh)
if not is_eps:
print("%%EOF", file=fh)
fh.flush()
if mpl.rcParams['ps.usedistiller']:
# We are going to use an external program to process the output.
# Write to a temporary file.
with TemporaryDirectory() as tmpdir:
tmpfile = os.path.join(tmpdir, "tmp.ps")
with open(tmpfile, 'w', encoding='latin-1') as fh:
print_figure_impl(fh)
if mpl.rcParams['ps.usedistiller'] == 'ghostscript':
_try_distill(gs_distill,
tmpfile,
is_eps,
ptype=papertype,
bbox=bbox)
elif mpl.rcParams['ps.usedistiller'] == 'xpdf':
_try_distill(xpdf_distill,
tmpfile,
is_eps,
ptype=papertype,
bbox=bbox)
_move_path_to_path_or_stream(tmpfile, outfile)
else:
# Write directly to outfile.
if passed_in_file_object:
requires_unicode = file_requires_unicode(outfile)
if not requires_unicode:
fh = TextIOWrapper(outfile, encoding="latin-1")
# Prevent the TextIOWrapper from closing the underlying
# file.
fh.close = lambda: None
else:
fh = outfile
print_figure_impl(fh)
else:
with open(outfile, 'w', encoding='latin-1') as fh:
print_figure_impl(fh)
def summary(path):
from zipfile import ZipFile
from pathlib import Path
from csv import DictReader
from sys import argv
from io import TextIOWrapper
from colorama import Style, Fore
class tree(object):
def __init__(self,
*labels,
test="",
status=None,
auto=False,
text="",
details=""):
if status:
stat = {
"pass": f"{Fore.GREEN}[PASS]",
"warn": f"{Fore.YELLOW}[WARN]",
"fail": f"{Fore.RED}[FAIL]"
}.get(status, "")
label = f"{stat}{Style.RESET_ALL} {text}"
else:
label = " ".join(labels)
self.label = label
self.children = []
for child in self.children:
if isinstance(child, tree):
self.children.append(child)
else:
self.children.append(tree(child))
def print(self, prefix=None, last=True):
if prefix is None:
print(self.label)
elif last:
print(prefix + " └─", self.label)
else:
print(prefix + " ├─", self.label)
for child in self.children:
if prefix is None:
child.print("", child is self.children[-1])
elif last:
child.print(prefix + " ", child is self.children[-1])
else:
child.print(prefix + " │ ", child is self.children[-1])
with (Path(path) / "report.zip").open("rb") as zdata, \
ZipFile(zdata) as zf :
infile = TextIOWrapper(zf.open("report.csv"),
encoding="utf-8",
errors="replace")
report = list(DictReader(infile))
root = tree(Path(argv[-1]).stem)
nodes = {"": root}
for test in report:
num = f".{test['test']}"
pid, cid = num.rsplit(".", 1)
nodes[num] = tree(**test)
nodes[pid].children.append(nodes[num])
root.print()
def save(self, commit=True, run_rule=False):
json_file = self.cleaned_data['json_file']
fc = TextIOWrapper(
json_file.file,
encoding=json_file.charset if json_file.charset else 'utf-8')
# an error message
import_errors = False
# the imported rule sets
success_rule_sets = []
# if the rulesets are run, count how many topics were affected
runner_count = 0
try:
rule_sets_data = json.loads(fc.read())
except json.decoder.JSONDecodeError:
import_errors = "Cannot parse JSON file."
else:
if rule_sets_data and rule_sets_data.get("tag_rule_sets"):
for tag_rule_set in rule_sets_data.get("tag_rule_sets"):
name = tag_rule_set.get("name")
if name:
# check if rule sets with given name is already exists
name = name.strip()
topic_tag_rule_sets = TopicTagRuleSet.objects.filter(
name=name)
if topic_tag_rule_sets:
for topic_tag_rule_set in topic_tag_rule_sets:
TopicTagRule.objects.filter(
rule_set=topic_tag_rule_set.id).delete()
topic_tag_rule_set.delete()
# import
topic_tag_rule_set = TopicTagRuleSet(name=name)
topic_tag_rule_set.save()
success_rule_sets.append(name)
rules = tag_rule_set.get("rules")
if rules:
for rule in rules:
name = rule.get("name")
if name:
topic_tag_rule = TopicTagRule(
name=name, rule_set=topic_tag_rule_set)
filters = rule.get("filters")
if filters:
topic_tag_rule.filters = filters
else:
topic_tag_rule.filters = []
tags = rule.get("tags")
if tags:
topic_tag_rule.tags = tags
else:
topic_tag_rule.tags = []
action = rule.get("action")
if action:
topic_tag_rule.action = action
action_fields = rule.get("action_fields")
if action_fields:
topic_tag_rule.action_fields = action_fields
topic_tag_rule.save()
if run_rule:
# run the imported ruleset
runner = TopicTagRuleSetRunForm(
{'ruleset_id': topic_tag_rule_set.id})
if runner.is_valid():
updated_set = runner.save()
runner_count += len(updated_set)
else:
import_errors = "JSON file rule sets is empty."
if import_errors:
return {'import_errors': import_errors}
elif success_rule_sets:
res = {'success_rule_sets': success_rule_sets}
if run_rule:
res['runner_count'] = runner_count
return res
else:
return {'import_errors': "Rule sets list to import is empty."}
def main():
params = inspect.signature(BufferedPipe.read).parameters
if "flags" not in params:
console.log(
f"[red]error[/red]: pwncat requires a custom fork of paramiko. This can be installed with `pip install -U git+https://github.com/calebstewart/paramiko`"
)
sys.exit(1)
# Ignore SQL Alchemy warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
# Default log-level is "INFO"
logging.getLogger().setLevel(logging.INFO)
# Build the victim object
pwncat.victim = Victim()
# Find the user configuration
config_path = (Path(os.environ.get("XDG_CONFIG_HOME", "~/.config/")) /
"pwncat" / "pwncatrc")
config_path = config_path.expanduser()
try:
# Read the config script
with config_path.open("r") as filp:
script = filp.read()
# Run the script
pwncat.victim.command_parser.eval(script, str(config_path))
except (FileNotFoundError, PermissionError):
# The config doesn't exist
pass
# Arguments to `pwncat` are considered arguments to `connect`
# We use the `prog_name` argument to make the help for "connect"
# display "pwncat" in the usage. This is just a visual fix, and
# isn't used anywhere else.
pwncat.victim.command_parser.dispatch_line(shlex.join(["connect"] +
sys.argv[1:]),
prog_name="pwncat")
# Only continue if we successfully connected
if not pwncat.victim.connected:
sys.exit(0)
# Make stdin unbuffered. Without doing this, some key sequences
# which are multi-byte don't get sent properly (e.g. up and left
# arrow keys)
sys.stdin = TextIOWrapper(
os.fdopen(sys.stdin.fileno(), "br", buffering=0),
write_through=True,
line_buffering=False,
)
# Setup the selector to wait for data asynchronously from both streams
selector = selectors.DefaultSelector()
selector.register(sys.stdin, selectors.EVENT_READ, None)
selector.register(pwncat.victim.client, selectors.EVENT_READ, "read")
# Initialize our state
done = False
try:
# This loop is only used to funnel data between the local
# and remote hosts when in raw mode. During the `pwncat`
# prompt, the main loop is handled by the CommandParser
# class `run` method.
while not done:
for k, _ in selector.select():
if k.fileobj is sys.stdin:
data = sys.stdin.buffer.read(64)
pwncat.victim.process_input(data)
else:
data = pwncat.victim.recv()
if data is None or len(data) == 0:
done = True
break
sys.stdout.buffer.write(data)
sys.stdout.flush()
except ConnectionResetError:
pwncat.victim.restore_local_term()
console.log(
"[yellow]warning[/yellow]: connection reset by remote host")
except SystemExit:
console.log("closing connection")
finally:
# Restore the shell
pwncat.victim.restore_local_term()
try:
# Make sure everything was committed
pwncat.victim.session.commit()
except InvalidRequestError:
pass
def save(self, commit=True):
prefix = self.cleaned_data['device_prefix']
file = self.cleaned_data['csv_file']
config = self.cleaned_data['config_file']
site_id = self.cleaned_data['site']
import_count = 0
error_count = 0
logger.info('TopicImportForm: with prefix %s and file %s / %s / %s',
prefix, file.name, file.content_type, file.size)
f = TextIOWrapper(file.file,
encoding=file.charset if file.charset else 'utf-8')
records = csv.DictReader(f)
logger.info('TopicImportForm: with prefix %s and file %s / %s / %s',
prefix, config.name, config.content_type, config.size)
fc = TextIOWrapper(
config.file,
encoding=config.charset if config.charset else 'utf-8')
config_data = fc.read()
import_errors = False
import_success = False
site = None
if site_id:
try:
site = Entity.objects.get(entity_id=site_id)
except TopicTagRuleSet.DoesNotExist:
import_errors = 'Could not get site {}'.format(site_id)
if site:
for row in records:
if 'Volttron Point Name' in row:
topic = '/'.join([prefix, row['Volttron Point Name']])
entity_id = utils.make_random_id(topic)
name = row['Volttron Point Name']
# create the topic if it does not exist in the Crate Database
Topic.ensure_topic_exists(topic)
# update or create the Data Point
try:
e = Entity.objects.get(topic=topic)
except Entity.DoesNotExist:
e = Entity(entity_id=entity_id, topic=topic)
e.add_tag('id', entity_id, commit=False)
e.add_tag('point', commit=False)
e.add_tag('dis', name, commit=False)
if site.kv_tags.get('id'):
e.add_tag('siteRef',
site.kv_tags.get('id'),
commit=False)
if row.get('Units'):
e.add_tag('unit', row['Units'], commit=False)
e.add_tag(Tag.bacnet_tag_prefix + 'prefix',
prefix,
commit=False)
# add all bacnet tags
for k, v in row.items():
field = k.lower()
field = field.replace(' ', '_')
if field == 'point_name':
field = 'reference_point_name'
if v:
e.add_tag(Tag.bacnet_tag_prefix + field,
v,
commit=False)
# add config file fields
try:
config_data_json = json.loads(config_data)
except Exception:
logging.error("Cannot parse bacnet_config json")
else:
for key in config_data_json.keys():
value = config_data_json.get(key)
field = key.lower()
field = field.replace(' ', '_')
if value:
if key == 'driver_config':
for key1 in value.keys():
value1 = value.get(key1)
field = key1.lower()
field = field.replace(' ', '_')
e.add_tag(Tag.bacnet_tag_prefix +
field,
value1,
commit=False)
else:
if key == 'interval':
e.add_tag(field, value, commit=False)
else:
e.add_tag(Tag.bacnet_tag_prefix +
field,
value,
commit=False)
logger.info(
'TopicImportForm: imported Data Point %s as %s with topic %s',
entity_id, name, topic)
e.save()
import_count += 1
else:
error_count += 1
logger.error(
'TopicImportView cannot import row, no point name found, %s',
row)
import_success = 'Imported {} topics and Data Points with prefix {}'.format(
import_count, prefix)
if error_count:
import_errors = 'Could not Import {} records.'.format(
error_count)
result = {}
if import_errors:
result['import_errors'] = import_errors
if import_success:
result['import_success'] = import_success
return result
exchanges = {
'N': 'NYSE',
'P': 'ARCA'
}
if __name__ == "__main__":
args = parser.parse_args()
ftp = FTP('ftp.nasdaqtrader.com')
ftp.login()
if args.nasdaq:
r = BytesIO()
ftp.retrbinary('RETR /SymbolDirectory/nasdaqlisted.txt', r.write)
r.seek(0)
wrapper = TextIOWrapper(r, encoding='utf-8')
for line in wrapper:
ticker, name, market_cat, is_test, status, lot_size, x,y = line.split('|')
if is_test == 'N':
print('NASDAQ:{}'.format(ticker))
if args.nyse or args.arca:
r = BytesIO()
ftp.retrbinary('RETR /SymbolDirectory/otherlisted.txt', r.write)
r.seek(0)
wrapper = TextIOWrapper(r, encoding='utf-8')
for line in wrapper:
line_split = line.split('|')
def get_handle(
path_or_buf: FilePathOrBuffer,
mode: str,
encoding: Optional[str] = None,
compression: CompressionOptions = None,
memory_map: bool = False,
is_text: bool = True,
errors: Optional[str] = None,
storage_options: StorageOptions = None,
) -> IOHandles:
"""
Get file handle for given path/buffer and mode.
Parameters
----------
path_or_buf : str or file handle
File path or object.
mode : str
Mode to open path_or_buf with.
encoding : str or None
Encoding to use.
compression : str or dict, default None
If string, specifies compression mode. If dict, value at key 'method'
specifies compression mode. Compression mode must be one of {'infer',
'gzip', 'bz2', 'zip', 'xz', None}. If compression mode is 'infer'
and `filepath_or_buffer` is path-like, then detect compression from
the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise
no compression). If dict and compression mode is one of
{'zip', 'gzip', 'bz2'}, or inferred as one of the above,
other entries passed as additional compression options.
.. versionchanged:: 1.0.0
May now be a dict with key 'method' as compression mode
and other keys as compression options if compression
mode is 'zip'.
.. versionchanged:: 1.1.0
Passing compression options as keys in dict is now
supported for compression modes 'gzip' and 'bz2' as well as 'zip'.
memory_map : boolean, default False
See parsers._parser_params for more information.
is_text : boolean, default True
Whether the type of the content passed to the file/buffer is string or
bytes. This is not the same as `"b" not in mode`. If a string content is
passed to a binary file/buffer, a wrapper is inserted.
errors : str, default 'strict'
Specifies how encoding and decoding errors are to be handled.
See the errors argument for :func:`open` for a full list
of options.
storage_options: StorageOptions = None
Passed to _get_filepath_or_buffer
.. versionchanged:: 1.2.0
Returns the dataclass IOHandles
"""
# Windows does not default to utf-8. Set to utf-8 for a consistent behavior
encoding_passed, encoding = encoding, encoding or "utf-8"
# read_csv does not know whether the buffer is opened in binary/text mode
if _is_binary_mode(path_or_buf, mode) and "b" not in mode:
mode += "b"
# open URLs
ioargs = _get_filepath_or_buffer(
path_or_buf,
encoding=encoding,
compression=compression,
mode=mode,
storage_options=storage_options,
)
handle = ioargs.filepath_or_buffer
handles: List[Buffer]
# memory mapping needs to be the first step
handle, memory_map, handles = _maybe_memory_map(handle, memory_map,
ioargs.encoding,
ioargs.mode, errors)
is_path = isinstance(handle, str)
compression_args = dict(ioargs.compression)
compression = compression_args.pop("method")
if compression:
# compression libraries do not like an explicit text-mode
ioargs.mode = ioargs.mode.replace("t", "")
# GZ Compression
if compression == "gzip":
if is_path:
assert isinstance(handle, str)
handle = gzip.GzipFile(
filename=handle,
mode=ioargs.mode,
**compression_args,
)
else:
handle = gzip.GzipFile(
fileobj=handle, # type: ignore[arg-type]
mode=ioargs.mode,
**compression_args,
)
# BZ Compression
elif compression == "bz2":
handle = bz2.BZ2File(
handle, # type: ignore[arg-type]
mode=ioargs.mode,
**compression_args,
)
# ZIP Compression
elif compression == "zip":
handle = _BytesZipFile(handle, ioargs.mode, **compression_args)
if handle.mode == "r":
handles.append(handle)
zip_names = handle.namelist()
if len(zip_names) == 1:
handle = handle.open(zip_names.pop())
elif len(zip_names) == 0:
raise ValueError(
f"Zero files found in ZIP file {path_or_buf}")
else:
raise ValueError("Multiple files found in ZIP file. "
f"Only one file per ZIP: {zip_names}")
# XZ Compression
elif compression == "xz":
handle = get_lzma_file(lzma)(handle, ioargs.mode)
# Unrecognized Compression
else:
msg = f"Unrecognized compression type: {compression}"
raise ValueError(msg)
assert not isinstance(handle, str)
handles.append(handle)
elif isinstance(handle, str):
# Check whether the filename is to be opened in binary mode.
# Binary mode does not support 'encoding' and 'newline'.
if ioargs.encoding and "b" not in ioargs.mode:
if errors is None and encoding_passed is None:
# ignore errors when no encoding is specified
errors = "replace"
# Encoding
handle = open(
handle,
ioargs.mode,
encoding=ioargs.encoding,
errors=errors,
newline="",
)
else:
# Binary mode
handle = open(handle, ioargs.mode)
handles.append(handle)
# Convert BytesIO or file objects passed with an encoding
is_wrapped = False
if is_text and (compression or _is_binary_mode(handle, ioargs.mode)):
handle = TextIOWrapper(
handle, # type: ignore[arg-type]
encoding=ioargs.encoding,
errors=errors,
newline="",
)
handles.append(handle)
# only marked as wrapped when the caller provided a handle
is_wrapped = not (isinstance(ioargs.filepath_or_buffer, str)
or ioargs.should_close)
handles.reverse() # close the most recently added buffer first
if ioargs.should_close:
assert not isinstance(ioargs.filepath_or_buffer, str)
handles.append(ioargs.filepath_or_buffer)
assert not isinstance(handle, str)
return IOHandles(
handle=handle,
created_handles=handles,
is_wrapped=is_wrapped,
is_mmap=memory_map,
compression=ioargs.compression,
)
