def createOrder(request):
try:
user = request.user
profile = Profile.objects.get(user=user)
request.data['buyer'] = profile.id
request.data['distance'] = getDistanceInMeters(request)
orderSerializer = InstantDeliveryOrderSerializer(data=request.data)
if orderSerializer.is_valid():
request.data['buyer'] = profile
order = orderSerializer.create(request.data)
order.save()
inWalletId = order.buyer.wallet.id
feePercentage = Configuration().getConfiguration(
"forwarding_fee_percentage_level_1"
)
TransactionManager.createInstantDeliveryOrderPaymentTransaction(
amount=order.priceRange,
order=order,
inWalletId=inWalletId,
feePercentage=feePercentage
)
return Response({'success': True, 'orderId': order.id},
status=status.HTTP_201_CREATED)
else:
return Response({'success': False, 'errors': orderSerializer.errors})
except Profile.DoesNotExist:
return Response({'success': False, 'error': 'profile.notfound'})
except Exception as e:
return Response({'success': False, 'errors': e.message})
def searchP2PForwarders(request):
user = request.user
profile = Profile.objects.get(user=user)
locationFrom = request.data.get('locationFrom')
locationFromList = Location.objects.filter(countryCode=locationFrom['countryCode'])
locationFrom = locationFromList.values_list('pk', flat=True)
locationTo = request.data.get('locationTo')
locationToList = Location.objects.filter(countryCode=locationTo['countryCode'])
locationTo = locationToList.values_list('pk', flat=True)
maxWeight = request.data.get('maxWeight')
maxSize = request.data.get('maxSize')
maxGoodValue = request.data.get('maxGoodValue')
if profile.currencySetting == settings.PROFILE_CURRENCY_SETTINGS[1][0]:
maxGoodValue = Service.eurToUsdConversion(maxGoodValue)
configMaxGoodValuePercentage = Configuration().getConfiguration("p2p_forwarder_maxgoodvalue_percentage")
services = []
if request.data.get('acceptedPacksFromPrivateOrCompany'):
acceptPackagesFromprivateOrCompanyQuery = Q(acceptedPacksFromPrivate=True)
else:
acceptPackagesFromprivateOrCompanyQuery = Q(acceptedPacksFromCompany=True)
servicesTmp = Service.objects.filter(
Q(
type=SERVICE_TYPES[0][0],
enabled=True,
locationFrom__in=list(locationFrom),
locationTos__in=list(locationTo),
maxSize__gte=maxSize,
maxWeight__gte=maxWeight,
)
& acceptPackagesFromprivateOrCompanyQuery
).exclude(profile=profile).distinct()
# Check Max Good Value in Service -> profile -> wallet
for s in servicesTmp:
wallet = Wallet.objects.get(
profile=s.profile
)
if wallet.deposit > 0:
walletMaxGoodValue = (configMaxGoodValuePercentage * wallet.deposit) / 100
if walletMaxGoodValue >= maxGoodValue:
services.append(s)
shippingWeight = 0.5
if maxWeight == 1:
shippingWeight = 7
elif maxWeight == 2:
shippingWeight = 8
for s in services:
originCountry = CountryManager.countryByCoutryCode(s.locationFrom.countryCode)['name']
destinationCountry = CountryManager.countryByCoutryCode(locationToList[0].countryCode)['name']
s.setLowestPrice(
shippingWeight=shippingWeight,
originCountry=originCountry,
destinationCountry=destinationCountry,
margin=s.partnerForwarderMargin
)
serviceSerializer = ServiceProtectedSerializer(services, many=True, context={'request': request})
return Response({'success': True, 'services': serviceSerializer.data})
def newsletter_registration(email):
configuration = Configuration.get_solo()
headers = {
"accept": "application/json",
"content-type": "application/json",
"api-key": settings.SIB_API_KEY,
}
data = {
"email": email,
# "attributes": {
# "FIRSTNAME": "",
# "LASTNAME": ""
# },
"includeListIds": [int(settings.SIB_NEWSLETTER_LIST_ID)],
"templateId": int(settings.SIB_NEWSLETTER_DOI_TEMPLATE_ID),
"redirectionUrl":
configuration.application_frontend_url + "?newsletter=confirmed",
"updateEnabled": True,
}
return requests.post(settings.SIB_CONTACT_DOI_ENDPOINT,
headers=headers,
data=json.dumps(data))
def courierPartnerCost(request):
try:
user = request.user
Profile.objects.get(user=user)
curierFeeAmount = Configuration().getConfiguration(
"dhl_partner_fee"
)
return Response({'success': True, 'curierFeeAmount': curierFeeAmount})
except Exception as e:
return Response({'success': False, 'error': e.message})
def configuration_professional_edit(request):
if Configuration.objects.filter(provider=request.user.profile).count():
c = Configuration.objects.filter(provider=request.user.profile)[:1]\
.get()
else:
c = Configuration()
c.provider = request.user.profile
if request.method == 'GET':
form = ConfigurationForm(instance=c)
context = {
'form': form,
}
return render(request, 'providers/configuration_professional.html',
context)
elif request.method == 'POST':
form = ConfigurationForm(request.POST, request.FILES, instance=c)
if form.is_valid():
form.save()
messages.success(request, 'Configuração atualizada com sucesso!')
return redirect('root:configuration_index')
return render(request, 'providers/configuration_professional.html',
{'form': form})
def completePaymentTransaction(cls, outWalletId, order):
outWallet = Wallet.objects.get(id=outWalletId)
paymentTransaction = PaymentTransaction.objects.get(order=order)
if paymentTransaction.status != 'HOLDING':
raise ValueError("Transaction not in HOLDING status")
paymentTransaction.status = 'COMPLETED'
paymentTransaction.info += '\nCompleted Payment of %s Bc to %s user wallet. (%s)' % (
paymentTransaction.amount, outWallet.profile.user.username,
django.utils.timezone.now())
if order.service.type == settings.SERVICE_TYPES[0][0] \
and order.service.addPartnerForwarder:
feePartner = Configuration().getConfiguration("dhl_partner_fee")
paymentTransaction.courierPartnerFeeAmount = feePartner
outWallet.credit -= feePartner
paymentTransaction.info += "\n (Courier Partner Fee %.2f)" % feePartner
paymentTransaction.save()
outWallet.credit += paymentTransaction.amount
outWallet.save()
def changelist_view(self, request, extra_context=None):
"""
show chart of answers per day
https://dev.to/danihodovic/integrating-chart-js-with-django-admin-1kjb
Corresponding template in templates/admin/stats/dailystat/change_list.html
"""
if request.POST.get("run_generate_daily_stats_script", False):
management.call_command("generate_daily_stats")
# custom form
current_field = str(
request.POST.get("field",
constants.AGGREGATION_FIELD_CHOICE_LIST[0]))
current_scale = str(
request.POST.get("scale",
constants.AGGREGATION_SCALE_CHOICE_LIST[0]))
current_since_date = str(
request.POST.get("since_date",
constants.AGGREGATION_SINCE_DATE_DEFAULT))
# Aggregate answers per day
# chart_data_query = DailyStat.objects.extra(select={"day": "date(date)"}) # sqlite
chart_data_query = DailyStat.objects.agg_timeseries(
current_field, scale=current_scale, since_date=current_since_date)
chart_data_list = list(chart_data_query)
# Serialize and attach the chart data to the template context
chart_data_json = json.dumps(chart_data_list, cls=DjangoJSONEncoder)
extra_context = extra_context or {
"configuration": Configuration.get_solo(),
"chart_data": chart_data_json,
"field_choice_list": constants.AGGREGATION_FIELD_CHOICE_LIST,
"current_field": current_field,
"scale_choice_list": constants.AGGREGATION_SCALE_CHOICE_LIST,
"current_scale": current_scale,
"since_date_min": constants.AGGREGATION_SINCE_DATE_DEFAULT,
"current_since_date": current_since_date,
}
# Call the superclass changelist_view to render the page
return super().changelist_view(request, extra_context=extra_context)
def changelist_view(self, request, extra_context=None):
"""
Corresponding template in templates/admin/api/question/change_list_with_import.html
"""
notion_questions_import_scope_choices = [(
scope_value,
scope_label,
"notion_questions_scope_" + str(scope_value) + "_last_imported",
) for (
scope_value,
scope_label,
) in api_constants.NOTION_QUESTIONS_IMPORT_SCOPE_CHOICES[1:]]
notion_questions_import_response = []
if request.POST.get("run_import_questions_from_notion_script", False):
out = StringIO()
scope = request.POST.get("run_import_questions_from_notion_script")
management.call_command("import_questions_from_notion",
scope,
stdout=out)
notion_questions_import_response = out.getvalue()
notion_questions_import_response = notion_questions_import_response.split(
"\n")
notion_questions_import_response = [
elem.split("///") if ("///" in elem) else elem
for elem in notion_questions_import_response
]
extra_context = extra_context or {
"configuration": Configuration.get_solo(),
"notion_questions_import_scope_choices":
notion_questions_import_scope_choices,
"notion_questions_import_response":
notion_questions_import_response,
}
# Call the superclass changelist_view to render the page
return super().changelist_view(request, extra_context=extra_context)
def orderPayment(request):
try:
order = Order.objects.get(id=request.data.get("orderId"))
inWalletId = order.profile.wallet.id
outWalletId = order.service.profile.wallet.id
feePercentage = Configuration().getConfiguration(
"forwarding_fee_percentage_level_1")
TransactionManager.createPaymentTransaction(
amount=order.totalPrice,
order=order,
inWalletId=inWalletId,
outWalletId=outWalletId,
feePercentage=feePercentage)
# Notification to forwarder
orderNotification = Notification()
orderNotification.email = True
orderNotification.alert = True
orderNotification.user = order.service.profile.user
orderLink = "%sorders/forwarder/%s" % (
orderNotification.getEmailLinkBaseUrl(), order.id)
orderNotification.setEmailData(
"New LWF Order",
"notifications/email/forwarder_order_new_status.html", {
'order': order,
'orderLink': orderLink
})
alertText = "Order #%d has been paid" % order.id
orderNotification.alertData = "%s|%s|%d" % (
alertText, "/orders/forwarder", order.id)
orderNotification.save()
return Response({'success': True})
except Exception as e:
return Response({'success': False, 'errors': e.message})
def processBtcTransactions(FreezedCurrency):
r = {
"processed_addresses": 0,
"created_transactions": [],
"errors_transactions": [],
"errors_addresses": []
}
allWallets = Wallet.objects.filter()
for wallet in allWallets:
for btcAddress in wallet.getWalletToCcAddresses(currency='BTC'):
r['processed_addresses'] += 1
atm = AtmBtc(btcAddress.address.address)
btcAddress.address.address = atm.clean(btcAddress.address.address)
try:
addressRemoteTransactions = atm.getTransactions()
except Exception, ex:
addressRemoteTransactions = []
r['errors_addresses'].append("%s" % (traceback.format_exc()))
if len(addressRemoteTransactions) > 0:
for art in addressRemoteTransactions:
if art.get("error"):
r['errors_addresses'].append(
"failed get data for address: %s" %
(btcAddress.address.address))
else:
if art['positive'] and art['confirmations'] > 0:
try:
new = False
try:
transaction = Transaction.objects.get(
hash=art['hash'],
currency="BTC",
address=btcAddress.address)
except Transaction.DoesNotExist:
previousCredit = wallet.credit
transaction = Transaction()
transaction.hash = art['hash']
transaction.date = art['date']
transaction.type = 'deposit'
transaction.currency = 'BTC'
transaction.freezedUsd = FreezedCurrency.usd
transaction.freezedEur = FreezedCurrency.eur
transaction.amount = art['value']
transaction.wallet = wallet
transaction.address = btcAddress.address
transaction.save()
# update wallet credit
feePercentage = Configuration(
).getConfiguration(
"btc_deposit_percentage")
wallet.depositIn(
FreezedCurrency, transaction,
art['value'],
'by AtmBtc found new tx: %s with positive amount: %s (BTC)'
% (art['hash'], art['value']),
feePercentage)
new = True
#all good
# create transactions info btc
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'Currency Rate Date: %s' % strftime(
FreezedCurrency.dateUpdated,
'%Y-%m-%d %H:%M:%S')
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'BTC -> USD'
transactionInfo.cost = FreezedCurrency.usd
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'BTC -> EUR'
transactionInfo.cost = FreezedCurrency.eur
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'Previous Credit'
transactionInfo.cost = previousCredit
transactionInfo.save()
transactionInfo = TransactionInfo()
wallet = Wallet.objects.get(id=wallet.id)
transactionInfo.transaction = transaction
transactionInfo.description = 'Current Credit'
transactionInfo.cost = wallet.credit
transactionInfo.save()
except Exception, ex:
transaction = None
r['errors_transactions'].append(
"failed insert for transaction: %s" %
(art['hash']))
if new:
if transaction:
if not any(x.hash == art['hash'] for x in
r['created_transactions']):
r['created_transactions'].append(
transaction)
# Admin Notification
adminNotification = Notification()
adminNotification.email = True
adminNotification.user = User.objects.get(
username="******")
adminNotification.setEmailData(
"New BTC deposit transaction confirmed",
"notifications/email/admin_email_new_deposit_transaction_confirmed.html",
{
'transaction': transaction,
})
def handle(self, *args, **options):
# init
start_time = time.time()
current_datetime = datetime.now()
current_datetime_string = current_datetime.strftime("%Y-%m-%d-%H-%M")
current_datetime_string_pretty = current_datetime.strftime("%Y-%m-%d %H:%M")
branch_name = f"update-stats-{current_datetime_string}"
pull_request_name = f"Update: stats ({current_datetime_string_pretty})"
# update configuration first
configuration = Configuration.get_solo()
configuration.github_stats_last_exported = timezone.now()
configuration.save()
print(
"--- Step 1 done : init (%s seconds) ---"
% round(time.time() - start_time, 1)
)
# update & commit stats files
try:
#####################################
# data/stats.yaml
start_time = time.time()
stats_dict = {
**utilities_stats.question_stats(),
**utilities_stats.quiz_stats(),
**utilities_stats.answer_stats(),
**utilities_stats.category_stats(),
**utilities_stats.tag_stats(),
**utilities_stats.contribution_stats(),
}
stats_yaml = yaml.safe_dump(stats_dict, allow_unicode=True, sort_keys=False)
stats_element = utilities_github.create_file_element(
file_path="data/stats.yaml", file_content=stats_yaml
)
print(
"--- Step 2.1 done : stats.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/difficulty-levels.yaml
start_time = time.time()
difficulty_levels_list = utilities_stats.difficulty_aggregate()
difficulty_levels_yaml = yaml.safe_dump(
difficulty_levels_list, allow_unicode=True, sort_keys=False
)
difficulty_levels_element = utilities_github.create_file_element(
file_path="data/difficulty-levels.yaml",
file_content=difficulty_levels_yaml,
)
print(
"--- Step 2.2 done : difficulty-levels.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/authors.yaml
start_time = time.time()
authors_list = utilities_stats.author_aggregate()
authors_yaml = yaml.safe_dump(
authors_list, allow_unicode=True, sort_keys=False
)
authors_element = utilities_github.create_file_element(
file_path="data/authors.yaml", file_content=authors_yaml
)
print(
"--- Step 2.3 done : authors.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/languages.yaml
start_time = time.time()
languages_list = utilities_stats.language_aggregate()
languages_yaml = yaml.safe_dump(
languages_list, allow_unicode=True, sort_keys=False
)
languages_element = utilities_github.create_file_element(
file_path="data/languages.yaml", file_content=languages_yaml
)
print(
"--- Step 2.4 done : languages.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/quiz-stats.yaml
start_time = time.time()
quiz_detail_stats_list = utilities_stats.quiz_detail_stats()
quiz_detail_stats_yaml = yaml.safe_dump(
quiz_detail_stats_list, allow_unicode=True, sort_keys=False
)
quiz_stats_element = utilities_github.create_file_element(
file_path="data/quiz-stats.yaml", file_content=quiz_detail_stats_yaml
)
print(
"--- Step 2.5 done : quiz-stats.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# update frontend file with timestamp
# frontend/src/constants.js
start_time = time.time()
old_frontend_constants_file_content = utilities_github.get_file(
file_path="frontend/src/constants.js",
)
new_frontend_constants_file_content_string = utilities.update_frontend_last_updated_datetime( # noqa
old_frontend_constants_file_content.decoded_content.decode(),
current_datetime_string_pretty,
)
new_frontend_constants_file_element = utilities_github.create_file_element(
file_path="frontend/src/constants.js",
file_content=new_frontend_constants_file_content_string,
)
print(
"--- Step 2.6 done : constants.js (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# commit files
start_time = time.time()
utilities_github.update_multiple_files(
branch_name=branch_name,
commit_message="Data: stats update",
file_element_list=[
stats_element,
difficulty_levels_element,
authors_element,
languages_element,
quiz_stats_element,
new_frontend_constants_file_element,
],
)
print(
"--- Step 3 done : committed to branch (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# create pull request
start_time = time.time()
if not settings.DEBUG:
# create pull request
pull_request_message = (
"Mise à jour des stats :"
"<ul>"
"<li>data/stats.yaml</li>"
"<li>data/difficulty-levels.yaml</li>"
"<li>data/authors.yaml</li>"
"<li>data/languages.yaml</li>"
"<li>data/tags.yaml</li>"
"<li>data/quiz-stats.yaml</li>"
"</ul>"
)
pull_request = utilities_github.create_pull_request(
pull_request_title=pull_request_name,
pull_request_message=pull_request_message,
branch_name=branch_name,
pull_request_labels="automerge",
)
print(
"--- Step 4 done : created Pull Request (%s seconds) ---"
% round(time.time() - start_time, 1)
)
# return
self.stdout.write(pull_request.html_url)
except Exception as e:
print(e)
self.stdout.write(str(e))
from django.utils import timezone
from django.core.management import BaseCommand
from core.models import Configuration
from stats.models import (
# QuestionAggStat,
QuestionAnswerEvent,
QuestionFeedbackEvent,
QuizAnswerEvent,
# QuizFeedbackEvent,
DailyStat,
)
from api.models import Question
configuration = Configuration.get_solo()
class Command(BaseCommand):
"""
Usage:
python manage.py generate_daily_stats
Daily stats
- total number of answers
- total number of answers from questions
- total number of answers from quizs
- total number of quizs played
- total number of feedbacks (like/dislike)
answers per hour ?
def get_object(self, queryset=None):
return Configuration.load()
def handle(self, *args, **options):
#########################################################
# Init
#########################################################
scope = options["scope"]
notion_questions_list = []
questions_ids_duplicate = []
questions_ids_missing = []
tags_created = []
questions_created = []
questions_updated = set()
questions_updates = {}
validation_errors = []
all_categories_list = list(Category.objects.all())
all_tags_name_list = list(Tag.objects.all().values_list("name",
flat=True))
#########################################################
# Fetch questions from Notion
#########################################################
start_time = time.time()
try:
notion_questions_list = utilities_notion.get_questions_table_rows()
except: # noqa
self.stdout.write("Erreur accès Notion. token_v2 expiré ?")
return
print(
"--- Step 1 done : fetch questions from Notion (%s seconds) ---" %
round(time.time() - start_time, 1))
#########################################################
# Check question ids (duplicates & missing)
#########################################################
start_time = time.time()
# order by notion_questions_list by id
notion_questions_list = sorted(
notion_questions_list,
key=lambda question: question.get_property("id") or 0)
# check if id duplicates
notion_questions_id_list = [
question.get_property("id") for question in notion_questions_list
if question.get_property("id")
]
questions_ids_duplicate = [
item for item, count in collections.Counter(
notion_questions_id_list).items() if count > 1
]
# check if id 'missing'
for n in range(1, notion_questions_id_list[-1]):
if n not in notion_questions_id_list:
questions_ids_missing.append(n)
print(
"--- Step 2 done : check question ids (duplicates & missing) : %s seconds ---"
% round(time.time() - start_time, 1))
#########################################################
# Loop on questions and create, update, store validation_errors
#########################################################
start_time = time.time()
# reduce scope because of timeouts on Heroku (30 seconds)
if scope:
min_question_id = 200 * (scope - 1)
max_question_id = 200 * scope
notion_questions_list_scope = notion_questions_list[
min_question_id:max_question_id]
else:
notion_questions_list_scope = notion_questions_list
print(f"processing {len(notion_questions_list_scope)} questions")
print(
f"First question id : {notion_questions_list_scope[0].get_property('id')}"
)
print(
f"Last question id : {notion_questions_list_scope[-1].get_property('id')}"
)
for notion_question_row in notion_questions_list_scope:
question_validation_errors = []
notion_question_tag_objects = []
notion_question_last_updated = notion_question_row.get_property(
"Last edited time").date() # noqa
# check question has id
if notion_question_row.get_property("id") is None:
question_validation_errors.append(
ValidationError({"id": "Question sans id. vide ?"}))
# ignore questions not updated recently
elif options.get("skip-old") and (
notion_question_last_updated <
(datetime.now().date() -
timedelta(days=SKIP_QUESTIONS_LAST_UPDATED_SINCE_DAYS))):
pass
else:
# build question_dict from notion_row
notion_question_dict = self.transform_notion_question_row_to_question_dict(
notion_question_row)
# cleanup relation category
# - check category exists
# error if unknown category : api.models.DoesNotExist: Category matching query does not exist. # noqa
if notion_question_dict["category"] is not None:
notion_question_category_id = next(
(c.id for c in all_categories_list
if c.name == notion_question_dict["category"]),
None,
)
if notion_question_category_id:
notion_question_dict[
"category_id"] = notion_question_category_id
del notion_question_dict["category"]
else:
question_validation_errors.append(
ValidationError({
"category":
f"Question {notion_question_dict['id']}."
f"Category '{notion_question_dict['category']}' inconnue"
}))
# cleanup relation tags
# - if no tags, notion returns [""]
# - check tags exist, create them if not
# - then delete the "tags" key, we will make the M2M join later
notion_question_tag_name_list = []
if notion_question_dict["tags"] != [""]:
notion_question_tag_name_list = [
tag for tag in notion_question_dict["tags"]
if not tag.startswith("Quiz")
]
new_tags = [
new_tag for new_tag in notion_question_tag_name_list
if new_tag not in all_tags_name_list
]
# create missing tags
if len(new_tags):
Tag.objects.bulk_create(
[Tag(name=new_tag) for new_tag in new_tags])
all_tags_name_list += new_tags
tags_created += new_tags
del notion_question_dict["tags"]
# create or update
# - if the question does not have validation_errors
if len(question_validation_errors):
validation_errors += question_validation_errors
# - if the question has been updated recently
elif options.get("skip-old") and (
notion_question_last_updated <
(datetime.now().date() -
timedelta(days=SKIP_QUESTIONS_LAST_UPDATED_SINCE_DAYS))):
pass
else:
# the question doesn't have errors : ready to create/update
try:
db_question, created = Question.objects.get_or_create(
id=notion_question_dict["id"],
defaults=notion_question_dict)
# store info
if created:
notion_question_tag_objects = Tag.objects.filter(
name__in=notion_question_tag_name_list)
db_question.tags.set(notion_question_tag_objects)
questions_created.append(db_question.id)
else:
questions_updates_key = f"Question {db_question.id}"
# update basic fields
question_changes_list = self.update_question(
db_question, notion_question_dict)
if len(question_changes_list):
if questions_updates_key in questions_updates:
questions_updates[
questions_updates_key] += question_changes_list
else:
questions_updates[
questions_updates_key] = question_changes_list
questions_updated.add(db_question.id)
# update tags
question_tag_changes_string = self.update_question_tags(
db_question, notion_question_tag_name_list)
if question_tag_changes_string:
if questions_updates_key in questions_updates:
questions_updates[
questions_updates_key].append(
question_tag_changes_string)
else:
questions_updates[questions_updates_key] = [
question_tag_changes_string
]
questions_updated.add(db_question.id)
except ValidationError as e:
validation_errors.append(
f"Question {notion_question_dict['id']}: {e}")
except IntegrityError as e:
validation_errors.append(
f"Question {notion_question_dict['id']}: {e}")
print("--- Step 3 done : loop on questions : %s seconds ---" %
round(time.time() - start_time, 1))
#########################################################
# Build and send stats
#########################################################
start_time = time.time()
# done
questions_notion_count = (
f"Nombre de questions dans Notion : {len(notion_questions_list)}")
questions_scope_count = f"Nombre de questions prises en compte ici : {len(notion_questions_list_scope)}" # noqa
questions_scope_count += f" (de id {notion_questions_list_scope[0].get_property('id')} à id {notion_questions_list_scope[-1].get_property('id')})" # noqa
questions_ids_duplicate_message = f"ids 'en double' : {', '.join([str(question_id) for question_id in questions_ids_duplicate])}" # noqa
questions_ids_missing_message = f"ids 'manquants' : {', '.join([str(question_id) for question_id in questions_ids_missing])}" # noqa
tags_created_message = f"Nombre de tags ajoutés : {len(tags_created)}"
if len(tags_created):
tags_created_message += (
"\n" +
f"Détails : {', '.join([str(tag_name) for tag_name in tags_created])}"
)
questions_created_message = (
f"Nombre de questions ajoutées : {len(questions_created)}")
if len(questions_created):
questions_created_message += (
"\n" +
f"Détails : {', '.join([str(question_id) for question_id in questions_created])}"
)
questions_updated_message = (
f"Nombre de questions modifiées : {len(questions_updated)}")
if len(questions_updates):
questions_updated_message += "\n" + "\n".join([
key + "///" + "///".join(questions_updates[key])
for key in questions_updates
])
# check if any published quiz have non-validated questions
published_quizs = Quiz.objects.prefetch_related(
"questions").published()
for pq in published_quizs:
pq_not_validated_questions = pq.questions_not_validated_list
for question in pq_not_validated_questions:
validation_errors.append(
f"Quiz {pq.id}: Question {question.id} is not validated but quiz is published"
)
validation_errors_message = "Erreurs : "
if len(validation_errors):
validation_errors_message += (
f"Erreurs : {len(validation_errors)}" + "\n" +
"\n".join([str(error) for error in validation_errors]))
Contribution.objects.create(
text="Erreur(s) lors de l'import",
description=validation_errors_message,
type="erreur application",
)
# if not settings.DEBUG and scope == 0:
# utilities_notion.add_import_stats_row(
# len(notion_questions_list),
# len(questions_created),
# len(questions_updated),
# )
print("--- Step 4 done : build and send stats : %s seconds ---" %
round(time.time() - start_time, 1))
#########################################################
# Update configuration
#########################################################
configuration = Configuration.get_solo()
if scope:
setattr(
configuration,
f"notion_questions_scope_{scope}_last_imported",
timezone.now(),
)
else:
for scope in constants.NOTION_QUESTIONS_IMPORT_SCOPE_LIST[1:]:
setattr(
configuration,
f"notion_questions_scope_{scope}_last_imported",
timezone.now(),
)
configuration.save()
self.stdout.write("\n".join([
">>> Info sur les questions",
questions_notion_count,
questions_scope_count,
questions_ids_duplicate_message,
questions_ids_missing_message,
"",
">>> Info sur les tags ajoutés",
tags_created_message,
"",
">>> Info sur les questions ajoutées",
questions_created_message,
"",
">>> Info sur les questions modifiées",
questions_updated_message,
"",
">>> Erreurs lors de l'import",
validation_errors_message,
]))
def save_model(self, request, obj, form, change):
try:
rulesets = list()
rulesets_id = list()
if request.POST.getlist('rulesets'):
rulesets_id = request.POST.getlist('rulesets')
for ruleset_id in rulesets_id:
rulesets.append(RuleSetSuricata.get_by_id(ruleset_id))
# URL HTTP
if obj.method.name == "URL HTTP":
obj.save()
if obj.scheduled_rules_deployment_enabled and obj.scheduled_rules_deployment_crontab:
create_download_from_http_task(obj)
if obj.scheduled_deploy:
if rulesets:
for ruleset in rulesets:
try:
for probe in ruleset.suricata_set.all():
schedule = add_1_hour(obj.scheduled_rules_deployment_crontab)
schedule, _ = CrontabSchedule.objects.\
get_or_create(minute=schedule.minute,
hour=schedule.hour,
day_of_week=schedule.day_of_week,
day_of_month=schedule.day_of_month,
month_of_year=schedule.month_of_year,
)
schedule.save()
create_deploy_rules_task(probe, schedule, obj)
except Exception as e: # pragma: no cover
logger.exception(str(e))
download_from_http.delay(obj.uri, rulesets_id=rulesets_id)
messages.add_message(request, messages.SUCCESS, mark_safe("Upload source in progress. " +
"<a href='/admin/core/job/'>View Job</a>"))
# Upload file
elif obj.method.name == "Upload file":
obj.uri = str(time.time()) + "_to_delete"
obj.save()
count_signature_created, count_signature_updated, count_script_created, count_script_updated = \
obj.download_from_file(request.FILES['file'].name, rulesets)
message = 'File uploaded successfully : ' + str(count_signature_created) + \
' signature(s) created and ' + str(count_signature_updated) + \
' signature(s) updated - ' + str(count_script_created) + \
' script(s) created and ' + str(count_script_updated) + ' script(s) updated'
logger.debug("Upload file: " + str(message))
messages.add_message(request, messages.SUCCESS, message)
# MISP
elif obj.method.name == "MISP":
obj.uri = CoreConfiguration.get_value("MISP_HOST")
obj.save()
logger.debug("Uploading rules from MISP")
download_from_misp.delay(obj.uri, rulesets_id=rulesets_id)
messages.add_message(request, messages.SUCCESS,
mark_safe("Upload source in progress. " +
"<a href='/admin/core/job/'>View Job</a>"))
else: # pragma: no cover
logger.error('Upload method unknown : ' + obj.method.name)
messages.add_message(request, messages.ERROR, 'Upload method unknown : ' + obj.method.name)
except Exception as e: # pragma: no cover
logger.exception(str(e))
messages.add_message(request, messages.ERROR, str(e))
finally:
if os.path.isfile(settings.BASE_DIR + "/" + obj.file.name):
os.remove(settings.BASE_DIR + "/" + obj.file.name)
def handle(self, *args, **options):
# init
start_time = time.time()
current_datetime = datetime.now()
current_datetime_string = current_datetime.strftime("%Y-%m-%d-%H-%M")
current_datetime_string_pretty = current_datetime.strftime(
"%Y-%m-%d %H:%M")
branch_name = f"update-data-{current_datetime_string}"
pull_request_name = f"Update: data ({current_datetime_string_pretty})"
# update configuration first
configuration = Configuration.get_solo()
configuration.github_data_last_exported = timezone.now()
configuration.save()
print("--- Step 1 done : init (%s seconds) ---" %
round(time.time() - start_time, 1))
# update & commit data files
try:
#####################################
# data/configuration.yaml
start_time = time.time()
configuration_yaml = utilities.serialize_model_to_yaml(
"core", model_label="configuration", flat=True)
configuration_element = utilities_github.create_file_element(
file_path="data/configuration.yaml",
file_content=configuration_yaml)
print("--- Step 2.1 done : configuration.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# # data/categories.yaml
start_time = time.time()
# categories_yaml = utilities.serialize_model_to_yaml("api", model_label="category", flat=True) # noqa
# categories_element = utilities_github.create_file_element(
# file_path="data/categories.yaml",
# file_content=categories_element
# )
print("--- Step 2.2 done : categories.yaml (skipped) ---")
#####################################
# data/tags.yaml
start_time = time.time()
tags_yaml = utilities.serialize_model_to_yaml("api",
model_label="tag",
flat=True)
tags_element = utilities_github.create_file_element(
file_path="data/tags.yaml", file_content=tags_yaml)
print("--- Step 2.3 done : tags.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/questions.yaml
start_time = time.time()
questions_yaml = utilities.serialize_model_to_yaml(
"api", model_label="question", flat=True)
questions_element = utilities_github.create_file_element(
file_path="data/questions.yaml", file_content=questions_yaml)
print("--- Step 2.4 done : questions.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quizzes.yaml
start_time = time.time()
quizzes_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quiz", flat=True)
quizzes_element = utilities_github.create_file_element(
file_path="data/quizzes.yaml", file_content=quizzes_yaml)
print("--- Step 2.5 done : quizzes.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quiz-questions.yaml
start_time = time.time()
quiz_questions_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quizquestion", flat=True)
quiz_questions_element = utilities_github.create_file_element(
file_path="data/quiz-questions.yaml",
file_content=quiz_questions_yaml)
print("--- Step 2.6 done : quiz-questions.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quiz-relationships.yaml
start_time = time.time()
quiz_relationships_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quizrelationship", flat=True)
quiz_relationships_element = utilities_github.create_file_element(
file_path="data/quiz-relationships.yaml",
file_content=quiz_relationships_yaml,
)
print(
"--- Step 2.7 done : quiz-relationships.yaml (%s seconds) ---"
% round(time.time() - start_time, 1))
#####################################
# update frontend file with timestamp
# frontend/src/constants.js
start_time = time.time()
old_frontend_constants_file_content = utilities_github.get_file(
file_path="frontend/src/constants.js", )
new_frontend_constants_file_content_string = utilities.update_frontend_last_updated_datetime( # noqa
old_frontend_constants_file_content.decoded_content.decode(),
current_datetime_string_pretty,
)
new_frontend_constants_file_element = utilities_github.create_file_element(
file_path="frontend/src/constants.js",
file_content=new_frontend_constants_file_content_string,
)
print("--- Step 2.8 done : constants.js (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# commit files
start_time = time.time()
utilities_github.update_multiple_files(
branch_name=branch_name,
commit_message="Data: data update",
file_element_list=[
configuration_element,
tags_element,
questions_element,
quizzes_element,
quiz_questions_element,
quiz_relationships_element,
new_frontend_constants_file_element,
],
)
print("--- Step 3 done : committed to branch (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# create pull request
start_time = time.time()
if not settings.DEBUG:
# create pull request
pull_request_message = ("Mise à jour de la donnée :"
"<ul>"
"<li>data/configuration.yaml</li>"
"<li>data/tags.yaml</li>"
"<li>data/questions.yaml</li>"
"<li>data/quizzes.yaml</li>"
"<li>data/quiz-questions.yaml</li>"
"<li>data/quiz-relationships.yaml</li>"
"</ul>")
pull_request = utilities_github.create_pull_request(
pull_request_title=pull_request_name,
pull_request_message=pull_request_message,
branch_name=branch_name,
pull_request_labels="automerge",
)
print(
"--- Step 4 done : created Pull Request (%s seconds) ---" %
round(time.time() - start_time, 1))
# return
self.stdout.write(pull_request.html_url)
except Exception as e:
print(e)
self.stdout.write(str(e))
def depositPaypal(request):
user = request.user
paymentId = request.data.get('paymentId')
payerId = request.data.get('payerId')
# print(payerId)
try:
profile = Profile.objects.get(user=user)
wallet = Wallet.objects.get(profile=profile)
previousCredit = wallet.credit
authUrl = settings.PAYPAL_GET_AUTH_TOKEN_ENDPOINT
headers = {
'Accept': 'application/json',
}
authResponse = requests.post(authUrl,
headers=headers,
data={
'grant_type': 'client_credentials'
},
auth=HTTPBasicAuth(
settings.PAYPAL_MERCHANT_ID,
settings.PAYPAL_SECRET)).json()
print(authResponse['access_token'])
headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer %s' % authResponse['access_token']
}
paymentUrl = settings.PAYPAL_PAYMENT_DETAILS_ENDPOINT % paymentId
paymentResponse = requests.get(paymentUrl, headers=headers).json()
# print(paymentResponse)
if 'error' in paymentResponse.keys():
return Response({
'success': False,
'error': paymentResponse['error_description']
})
if 'state' in paymentResponse.keys(
) and paymentResponse['state'] == 'approved':
paypalTransaction = paymentResponse['transactions'][0]
amount = float(paypalTransaction['amount']['total'])
currency = paypalTransaction['amount']['currency']
transaction = TransactionPaypal()
transaction.hash = paymentResponse['id']
transaction.date = paymentResponse['create_time']
transaction.type = 'deposit'
transaction.currency = currency
transaction.amount = amount
transaction.wallet = wallet
feePercentage = Configuration().getConfiguration(
"paypal_deposit_percentage")
freezedCurrency = Currency().getCurrency('PPBC')
if currency == 'USD':
pass
elif currency == 'EUR':
amount = (amount / freezedCurrency.eur)
else:
return Response({
'success':
False,
'error':
'currency.notavailable',
'message':
'deposit of %s in %s not available' % (amount, currency)
})
transaction.paymentId = paymentId
transaction.save()
wallet.depositIn(
freezedCurrency,
transaction,
amount,
'Paypal transaction found ID: %s with positive amount: %s (USD)'
% (paymentId, amount),
feePercentage,
)
#all good
# create transactions info for paypal
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'Currency Rate Date: %s' % strftime(
freezedCurrency.dateUpdated, '%Y-%m-%d %H:%M:%S')
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'BC -> USD'
transactionInfo.cost = freezedCurrency.usd
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'BC -> EUR'
transactionInfo.cost = freezedCurrency.eur
transactionInfo.save()
transactionInfo = TransactionInfo()
transactionInfo.transaction = transaction
transactionInfo.description = 'Previous Credit'
transactionInfo.cost = previousCredit
transactionInfo.save()
transactionInfo = TransactionInfo()
profile = Profile.objects.get(user=user)
wallet = Wallet.objects.get(profile=profile)
transactionInfo.transaction = transaction
transactionInfo.description = 'Current Credit'
transactionInfo.cost = wallet.credit
transactionInfo.save()
return Response({'success': True})
return Response({'success': False})
except Profile.DoesNotExist:
return Response({'success': False, 'error': 'profile.notfound'})
except Wallet.DoesNotExist:
return Response({'success': False, 'error': 'wallet.notfound'})