def get_big_transactions(transactions: List[Transaction], ceil: float,
start: datetime, end: datetime):
analyser = Analyser(transactions)
analyser.filter_date(start, end).filter_gt_abs_ceil(ceil)
print(f"\nBig transactions between {start} and {end}\n")
print(analyser)
print("\n\n")
def get_positive_transactions(transactions: List[Transaction], start: datetime,
end: datetime):
analyser = Analyser(transactions)
analyser.filter_date(start, end).filter_negative_transaction()
print(f"\nPositive transactions between {start} and {end}\n")
print(analyser)
print("\n\n")
def perform(self, classifier, spellcheck, debug, sentences, respond_to):
sc = (spellcheck and SpellChecker()) or None
cl = RedisClassifier(sc)
results = []
an = Analyser(cl, debug)
for sentence in sentences:
result = {}
result["text"] = sentence["text"]
result["user_info"] = sentence["user_info"]
result["result"] = an.process(sentence["text"])
results.append(result)
if len(respond_to) > 0:
requests.post(respond_to, headers={'Content-Type': 'application/json'}, data=json.dumps(results))
else:
return results
def get_employer_payment(transactions: List[Transaction],
start: datetime,
end: datetime,
business_trip_only=False):
analyser = Analyser(transactions)
business_trip_label_deprecated = "VIR SEPA RECU /DE COULOMBEL SYLVAIN /MOTIF /REF "
business_trip_label = "DEUS SAS - DIRECTION FINANCIERE /MOTIF INV"
salary_label = "DEUS SAS - DIRECTION FINANCIERE /MOTIF /REF AMAD"
label_list = [business_trip_label_deprecated, business_trip_label]
if not business_trip_only:
label_list.append(salary_label)
analyser.filter_date(start,
end).filter_by_label_contains_value_list(label_list)
print(
f"\nEmployer transactions between {start} and {end} (with business trip only set to {business_trip_only})\n"
)
print(analyser)
print("\n\n")
def month_analysis(transactions: List[Transaction]):
""""
Top 5 transactions in the account + cb spent + top cb + Naturalia reduction + epsilon (recoupement)
enables to see where money is going
"""
print("============ Last months analysis ==============")
start, end = datetime(2020, 6, 15), datetime(2020, 7, 15)
print(f"\nAnalyse period is between {start} and {end}")
analyser = Analyser(transactions)
print(f"Top 5 transactions in the account are\n")
print(
analyser.filter_date(start, end).filter_positive_transaction().
sort_transactions_by_amount().head(5))
analyser.reset()
analyser.filter_date(start, end).filter_by_kind("PAIEMENT CB")
print(
f"\nCB Spent between {start} and {end} is {analyser.reduce_to_sum()}")
# print(analyser.highest_spent_transaction())
print(f"Where top CB transactions are\n")
print(analyser.sort_transactions_by_amount().head(2))
print("\nThe CB spent includes Naturalia =>")
analyse_by_label_content("NATURALIA", transactions, start, end)
def analyse_by_label_content(label: str, transactions: List[Transaction],
start: datetime, end: datetime) -> float:
""""
To print transaction
>>> print(analyser.__str__()[0:500])
>>> print(anylyser)
"""
analyser = Analyser(transactions)
analyser.filter_date(start, end).filter_by_label_contains_value(label)
total_spent_sum = analyser.reduce_to_sum()
average_basket = analyser.reduce_to_average()
average_spent_per_day = total_spent_sum / (end - start).days
print(f"{label} Analysis between {start} and {end}\n"
f"Total spent is: {total_spent_sum}\n"
f"Average basket is {average_basket}\n"
f"Average spent per day is: {average_spent_per_day}")
print("\n\n")
return total_spent_sum
#Uso: ./run_analyser.py "texto" -debug
import sys
from analyser.analyser import Analyser
from spell_checker.spell_checker import SpellChecker
from analyser.dummy_classifier import DummyClassifier
from analyser.redis_classifier import RedisClassifier
next_is_textfile = False
debug = False
text = None
for arg in sys.argv[1:]:
if next_is_textfile:
text = file(arg).read()
next_is_textfile = False
elif arg=="-debug":
debug = True
elif arg == "-f":
next_is_textfile = True
elif not text:
text = arg
sc = SpellChecker()
#cl = DummyClassifier(sc)
cl = RedisClassifier(sc)
an = Analyser(cl, debug)
result = an.process(text)
print "%s (%s)" % ("Positivo" if result > 0 else "Negativo" if result < 0 else "Neutro", result)
def analyse_upload(self, upload):
diskname = upload.diskname()
zipfile = None
file = None
try:
if upload.filename.endswith(
'.evtc.zip') or upload.filename.endswith('.zevtc'):
zipfile = ZipFile(diskname)
contents = zipfile.infolist()
if len(contents) == 1:
try:
file = zipfile.open(contents[0].filename)
except RuntimeError as e:
raise EvtcAnalysisException(e)
else:
raise EvtcParseException(
'Only single-file ZIP archives are allowed')
else:
file = open(diskname, 'rb')
evtc_encounter = EvtcEncounter(file)
analyser = Analyser(evtc_encounter)
dump = analyser.data
uploader = upload.uploaded_by
started_at = dump['Category']['encounter']['start']
duration = dump['Category']['encounter']['duration']
success = dump['Category']['encounter']['success']
boss_name = analyser.boss_info.name
upload_val = upload.val
area_id = evtc_encounter.area_id
minDuration = 60
if BOSSES[area_id].kind == Kind.FRACTAL:
minDuration = 30
if duration < minDuration:
raise EvtcAnalysisException('Encounter shorter than 60s')
if dump['Category']['encounter']['cm']:
boss_name += " (CM)"
if analyser.boss_info.non_cm_allowed:
area_id += 0xFF0000
era = Era.by_time(started_at)
area, _ = Area.objects.get_or_create(id=area_id,
defaults={"name": boss_name})
status_for = {
name: player
for name, player in dump[Group.CATEGORY]['status']
['Player'].items() if 'account' in player
}
account_names = [
player['account'] for player in status_for.values()
]
with transaction.atomic():
# heuristics to see if the encounter is a re-upload:
# a character can only be in one raid at a time
# account_names are being hashed, and the triplet
# (area, account_hash, started_at) is being checked for
# uniqueness (along with some fuzzing to started_at)
started_at_full, started_at_half = Encounter.calculate_start_guards(
started_at)
account_hash = Encounter.calculate_account_hash(account_names)
filename = upload.filename
orig_filename = filename
if not zipfile:
filename += ".zip"
try:
encounter = Encounter.objects.get(
Q(started_at_full=started_at_full)
| Q(started_at_half=started_at_half),
area=area,
account_hash=account_hash)
if encounter.has_evtc:
old_filename = encounter.diskname()
if old_filename:
try:
os.remove(old_filename)
except FileNotFoundError:
pass
encounter.era = era
encounter.filename = filename
encounter.uploaded_at = upload.uploaded_at
encounter.uploaded_by = upload.uploaded_by
encounter.duration = duration
encounter.success = success
encounter.val = dump
encounter.started_at = started_at
encounter.started_at_full = started_at_full
encounter.started_at_half = started_at_half
encounter.has_evtc = True
except Encounter.DoesNotExist:
encounter = Encounter.objects.create(
filename=filename,
uploaded_at=upload.uploaded_at,
uploaded_by=upload.uploaded_by,
duration=duration,
success=success,
val=dump,
area=area,
era=era,
started_at=started_at,
started_at_full=started_at_full,
started_at_half=started_at_half,
has_evtc=True,
account_hash=account_hash)
if 'category_id' in upload_val:
encounter.category_id = upload_val['category_id']
if 'tagstring' in upload_val:
encounter.tagstring = upload_val['tagstring']
encounter.save()
file.close()
file = None
new_diskname = encounter.diskname()
os.makedirs(os.path.dirname(new_diskname), exist_ok=True)
if zipfile:
zipfile.close()
zipfile = None
os.rename(diskname, new_diskname)
else:
with ZipFile(new_diskname, 'w',
ZIP_DEFLATED) as zipfile_out:
zipfile_out.write(diskname, orig_filename)
for name, player in status_for.items():
account, _ = Account.objects.get_or_create(
name=player['account'])
participation, _ = Participation.objects.update_or_create(
account=account,
encounter=encounter,
defaults={
'character': name,
'archetype': player['archetype'],
'profession': player['profession'],
'party': player['party'],
'elite': player['elite']
})
if account.user:
Notification.objects.create(
user=account.user,
val={
"type": "upload",
"upload_id": upload.id,
"uploaded_by": upload.uploaded_by.username,
"filename": upload.filename,
"encounter_id": encounter.id,
"encounter_url_id": encounter.url_id,
"encounter": participation.data(),
})
if uploader and account.user_id == uploader.id:
uploader = None
if uploader:
Notification.objects.create(user=uploader,
val={
"type":
"upload",
"upload_id":
upload.id,
"uploaded_by":
upload.uploaded_by.username,
"filename":
upload.filename,
"encounter_id":
encounter.id,
"encounter_url_id":
encounter.url_id,
})
# if self.gdrive_service:
# media = MediaFileUpload(new_diskname, mimetype='application/prs.evtc')
# try:
# if encounter.gdrive_id:
# result = self.gdrive_service.files().update(
# fileId=encounter.gdrive_id,
# media_body=media,
# ).execute()
# else:
# metadata = {
# 'name': upload.filename,
# 'parents': [gdrive_folder],
# }
# gdrive_file = self.gdrive_service.files().create(
# body=metadata, media_body=media,
# fields='id, webContentLink',
# ).execute()
# encounter.gdrive_id = gdrive_file['id']
# encounter.gdrive_url = gdrive_file['webContentLink']
# encounter.save()
# except HttpError as e:
# logger.error(e)
# pass
#
logger.info("saved")
except (EvtcParseException, EvtcAnalysisException, BadZipFile) as e:
logger.info("known error: %s" % str(e))
Notification.objects.create(user=upload.uploaded_by,
val={
"type": "upload_error",
"upload_id": upload.id,
"error": str(e),
})
# for diagnostics and catching new exceptions
except Exception as e:
logger.info("unknown error: %s" % str(e))
exc = format_exc()
path = os.path.join(upload_dir, 'errors')
os.makedirs(path, exist_ok=True)
path = os.path.join(path, os.path.basename(diskname))
os.rename(diskname, path)
with open(path + '.error', 'w') as f:
f.write("%s (%s)\n" %
(upload.filename, upload.uploaded_by.username))
f.write(exc)
logger.error(exc)
Notification.objects.create(
user=upload.uploaded_by,
val={
"type":
"upload_error",
"upload_id":
upload.id,
"error":
"An unexpected error has occured, and your file has been stored for inspection by the developers.",
})
finally:
if file:
file.close()
if zipfile:
zipfile.close()
upload.delete()
