PEERINGDB = "https://peeringdb.com/asn/"
PEERINGDB_USERNAME = getattr(configuration, "PEERINGDB_USERNAME", "")
PEERINGDB_PASSWORD = getattr(configuration, "PEERINGDB_PASSWORD", "")
# GitHub releases check
RELEASE_CHECK_URL = getattr(
configuration,
"RELEASE_CHECK_URL",
"https://api.github.com/repos/peering-manager/peering-manager/releases",
)
RELEASE_CHECK_TIMEOUT = getattr(configuration, "RELEASE_CHECK_TIMEOUT", 86400)
# Validate repository URL and timeout
if RELEASE_CHECK_URL:
try:
URLValidator(RELEASE_CHECK_URL)
except ValidationError:
raise ImproperlyConfigured(
"RELEASE_CHECK_URL must be a valid API URL. Example: https://api.github.com/repos/peering-manager/peering-manager"
)
if RELEASE_CHECK_TIMEOUT < 3600:
raise ImproperlyConfigured(
"RELEASE_CHECK_TIMEOUT must be at least 3600 seconds (1 hour)")
try:
from peering_manager.ldap_config import *
LDAP_CONFIGURED = True
except ImportError:
LDAP_CONFIGURED = False
def clean(self):
url_with_scheme = 'https://' + self.url
validator = URLValidator()
validator(url_with_scheme)
class Project(models.Model):
name = models.CharField(max_length=40)
url = models.URLField(validators=[URLValidator(schemes=['http', 'https'])])
def __str__(self):
return self.name
def clean_url(self):
if self.cleaned_data['url']:
validator = URLValidator()
validator(self.cleaned_data['url'])
return self.cleaned_data['url']
def url_validator(url_str):
URLValidator(url_str)
return url_str
def update_local_db_based_on_record(eox_record, create_missing=False):
"""
update a database entry based on an EoX record provided by the Cisco EoX API
:param eox_record: JSON data from the Cisco EoX API
:param create_missing: set to True, if the product should be created if it's not part of the local database
:return: returns an error message or None if successful
"""
pid = eox_record['EOLProductID']
if create_missing:
product, created = Product.objects.get_or_create(product_id=pid)
else:
try:
product = Product.objects.get(product_id=pid)
created = False
except ObjectDoesNotExist:
logger.debug("%15s: Product not found in database (create disabled)" % pid, exc_info=True)
return None
if created:
product.product_id = pid
product.description = eox_record['ProductIDDescription']
# it is a Cisco API and the vendors are predefined in the database
product.vendor = Vendor.objects.get(name="Cisco Systems")
logger.debug("%15s: Product created" % pid)
# update the lifecycle information
try:
logger.debug("%15s: update product lifecycle values" % pid)
# save datetime values from Cisco EoX API record
value_map = {
# <API value> : <class attribute>
"UpdatedTimeStamp": "eox_update_time_stamp",
"EndOfSaleDate": "end_of_sale_date",
"LastDateOfSupport": "end_of_support_date",
"EOXExternalAnnouncementDate": "eol_ext_announcement_date",
"EndOfSWMaintenanceReleases": "end_of_sw_maintenance_date",
"EndOfRoutineFailureAnalysisDate": "end_of_routine_failure_analysis",
"EndOfServiceContractRenewal": "end_of_service_contract_renewal",
"EndOfSvcAttachDate": "end_of_new_service_attachment_date",
"EndOfSecurityVulSupportDate": "end_of_sec_vuln_supp_date",
}
for key in value_map.keys():
if eox_record.get(key, None):
value = eox_record[key].get("value", None)
value = value.strip() if value else ""
if value != "":
setattr(
product,
value_map[key],
datetime.strptime(
value,
convert_time_format(eox_record[key].get("dateFormat", "%Y-%m-%d"))
).date()
)
# save string values from Cisco EoX API record
if "LinkToProductBulletinURL" in eox_record.keys():
value = clean_api_url_response(eox_record.get('LinkToProductBulletinURL', ""))
if value != "":
val = URLValidator()
try:
val(value)
product.eol_reference_url = value
except ValidationError:
raise Exception("invalid EoL reference URL")
if "ProductBulletinNumber" in eox_record.keys():
product.eol_reference_number = eox_record.get('ProductBulletinNumber', "EoL bulletin")
with transaction.atomic(), reversion.create_revision():
product.save()
reversion.set_comment("Updated by the Cisco EoX API crawler")
except Exception as ex:
if created:
# remove the new (incomplete) entry from the database
product.delete()
logger.error("%15s: Product Data update failed." % pid, exc_info=True)
logger.debug("%15s: DataSet with exception\n%s" % (pid, json.dumps(eox_record, indent=4)))
return "Product Data update failed: %s" % str(ex)
# save migration information if defined
if "EOXMigrationDetails" in eox_record:
migration_details = eox_record["EOXMigrationDetails"]
product_migration_source, created = ProductMigrationSource.objects.get_or_create(
name="Cisco EoX Migration option"
)
if created:
product_migration_source.description = "Migration option suggested by the Cisco EoX API."
product_migration_source.save()
if "MigrationOption" in migration_details:
candidate_replacement_pid = migration_details["MigrationProductId"].strip()
if candidate_replacement_pid == pid:
logger.error("Product ID '%s' should be replaced by itself, which is not possible" % pid)
else:
# only a single migration option per migration source is allowed
pmo, _ = ProductMigrationOption.objects.get_or_create(product=product,
migration_source=product_migration_source)
if migration_details["MigrationOption"] == "Enter PID(s)":
# product replacement available, add replacement PID
pmo.replacement_product_id = candidate_replacement_pid
pmo.migration_product_info_url = clean_api_url_response(migration_details["MigrationProductInfoURL"])
elif migration_details["MigrationOption"] == "See Migration Section" or \
migration_details["MigrationOption"] == "Enter Product Name(s)":
# complex product migration, only add comment
mig_strat = migration_details["MigrationStrategy"].strip()
pmo.comment = mig_strat if mig_strat != "" else migration_details["MigrationProductName"].strip()
pmo.migration_product_info_url = clean_api_url_response(migration_details["MigrationProductInfoURL"])
else:
# no replacement available, only add comment
pmo.comment = migration_details["MigrationOption"].strip() # some data separated by blank
pmo.migration_product_info_url = clean_api_url_response(migration_details["MigrationProductInfoURL"])
# add message if only a single entry was saved
if pmo.migration_product_info_url != migration_details["MigrationProductInfoURL"].strip():
return "Multiple URL values from the Migration Note received, only the first one is saved"
pmo.save()
def marketing_link(name):
"""Returns the correct URL for a link to the marketing site
depending on if the marketing site is enabled
Since the marketing site is enabled by a setting, we have two
possible URLs for certain links. This function is to decides
which URL should be provided.
"""
# link_map maps URLs from the marketing site to the old equivalent on
# the Django site
link_map = settings.MKTG_URL_LINK_MAP
enable_mktg_site = configuration_helpers.get_value(
'ENABLE_MKTG_SITE', settings.FEATURES.get('ENABLE_MKTG_SITE', False))
marketing_urls = configuration_helpers.get_value('MKTG_URLS',
settings.MKTG_URLS)
marketing_url_overrides = configuration_helpers.get_value(
'MKTG_URL_OVERRIDES', settings.MKTG_URL_OVERRIDES)
if name in marketing_url_overrides:
validate = URLValidator()
url = marketing_url_overrides.get(name)
try:
validate(url)
return url
except ValidationError as err:
log.debug("Invalid link set for link %s: %s", name, err)
return '#'
if enable_mktg_site and name in marketing_urls:
# special case for when we only want the root marketing URL
if name == 'ROOT':
return marketing_urls.get('ROOT')
# special case for new enterprise marketing url with custom tracking query params
if name == 'ENTERPRISE':
enterprise_url = marketing_urls.get(name)
# if url is not relative, then return it without joining to root
if not enterprise_url.startswith('/'):
return enterprise_url
# Using urljoin here allows us to enable a marketing site and set
# a site ROOT, but still specify absolute URLs for other marketing
# URLs in the MKTG_URLS setting
# e.g. urljoin('http://marketing.com', 'http://open-edx.org/about') >>> 'http://open-edx.org/about'
return urljoin(marketing_urls.get('ROOT'), marketing_urls.get(name))
# only link to the old pages when the marketing site isn't on
elif not enable_mktg_site and name in link_map:
# don't try to reverse disabled marketing links
if link_map[name] is not None:
host_name = get_current_request_hostname()
if link_map[name].startswith('http'):
return link_map[name]
else:
try:
return reverse(link_map[name])
except NoReverseMatch:
log.debug(u"Cannot find corresponding link for name: %s",
name)
set_custom_attribute('unresolved_marketing_link', name)
return '#'
else:
log.debug(u"Cannot find corresponding link for name: %s", name)
return '#'
if value is None or not value.strip():
raise ValidationValueError('Title cannot be blank.')
value = strip_html(value)
if not allow_blank:
if value is None or not value.strip():
raise ValidationValueError('Invalid title.')
if len(value) > 512:
raise ValidationValueError('Title cannot exceed 512 characters.')
return True
validate_url = URLValidator()
def validate_profile_websites(profile_websites):
for value in profile_websites or []:
try:
validate_url(value)
except DjangoValidationError:
# Reraise with a better message
raise ValidationError('Invalid personal URL.')
def validate_social(value):
validate_profile_websites(value.get('profileWebsites'))
from osf.models import OSFUser
for soc_key in value.keys():
class ELib:
_dels = " ['\"“”‘’]+|[.?!,…]+|[:;]+(?:--+|―|—|~|–|=)@^*&%$#{}<>/\\~"
tokenizer = TweetTokenizer()
url_checker = URLValidator()
__rnd_print_counter = 0
@staticmethod
def __out(force, text, end):
printAll = True
if printAll or force:
print(text, end=end)
@staticmethod
def outLine(line):
ELib.__out(False, line, "\n")
@staticmethod
def outLineForce(line):
ELib.__out(True, line, "\n")
@staticmethod
def out(text):
ELib.__out(False, text, "")
@staticmethod
def outForce(text):
ELib.__out(True, text, "")
@staticmethod
def normalizeTime(text):
if not str.isdigit(text[0]):
# tokens = text.split(" ")
# result = tokens[3] + " - " + tokens[2] + " " + tokens[1] + " " + tokens[5]
# return result
return text
else:
return text
@staticmethod
def get_time(only_time=False):
if only_time:
return datetime.now().strftime('%H:%M:%S')
return datetime.now().strftime('%Y/%m/%d %H:%M:%S')
@staticmethod
def is_delimiter(str):
for dl in ELib._dels:
if str == dl:
return True
return False
@staticmethod
def PASS():
pass
@staticmethod
def progress_made(current_index, max_value_exclusive, number_of_intervals):
current_number = current_index + 1
result_1 = (current_number == 1)
result_2 = int((number_of_intervals * current_number) / max_value_exclusive) !=\
int((number_of_intervals * (current_number - 1)) / max_value_exclusive)
result = result_1 or result_2
return result
@staticmethod
def progress_percent(current_index, max_value_exclusive):
current_number = current_index + 1
result = ((10.0 * current_number) / max_value_exclusive) * 10
return str(int(result)) + '%'
@staticmethod
def __is_number(param):
try:
float(param)
except ValueError:
return False
return True
@staticmethod
def tokenize_tweet_text(tw,
normalize,
tokenize_by_etokens=False,
pivot_query=None,
query_list=None):
if tokenize_by_etokens:
tokens = list()
for to_ind, tok in enumerate(tw.ETokens):
temp = unidecode.unidecode(tok.Text).lower().replace(' ', '-')
if temp != '':
tok.Text = temp
tokens.append(temp)
else:
tok.Text = '.'
tokens.append('.')
else:
tokens = ELib.tokenizer.tokenize(tw.Text)
for to_ind, cur_tok in enumerate(tokens):
if pivot_query is not None:
if cur_tok.lower() in query_list:
tokens[to_ind] = pivot_query
if normalize:
if re.match(ELib.url_checker.regex, cur_tok) is not None:
tokens[to_ind] = 'www'
# elif cur_tok.startswith('@'):
# tokens[to_ind] = '@user'
# elif ELib.__is_number(cur_tok):
# tokens[to_ind] = 'number'
result = (' '.join(tokens)).lower()
return result
@staticmethod
def draw_activation_curves(curves, names, start_x=0):
# draw_curve([
# [[ndarray], [ndarray], [ndarray]],
# [[ndarray], [ndarray], [ndarray]]
# ],
# ['c1',
# 'c2'])
# scenario: show the output activations of multiple layers across multiple epochs
fig = plt.figure()
for s_ind, cur_series in enumerate(curves):
x = list()
means = list()
vars = list()
for t_ind, cur_t in enumerate(cur_series):
x.append(start_x + t_ind)
means.append(np.mean(cur_t))
vars.append(np.std(cur_t))
plt.errorbar(x=x, y=means, yerr=vars, label=names[s_ind])
assert 3 == 2 + 1
plt.legend(fontsize=20, loc='lower right')
plt.show()
assert 3 == 2 + 1
@staticmethod
def draw_gradient_curves(curves, names):
# draw_curve([
# [ndarray],
# [ndarray]
# ],
# ['c1',
# 'c2'])
# scenario 1: show the gradient distributions of one specific layer across multiple epochs
# scenario 2: show the gradient distributions of multiple layers in one specific epoch
fig = plt.figure()
for s_ind, cur_series in enumerate(curves):
y, binEdges = np.histogram(cur_series, bins=100, normed=True)
bincenters = 0.5 * (binEdges[1:] + binEdges[:-1])
plt.plot(bincenters, y, '-', label=names[s_ind])
assert 3 == 2 + 1
plt.legend(fontsize=20, loc='upper right')
plt.show()
assert 3 == 2 + 1
@staticmethod
def average_lists_elementwise(param_list):
param_list_np = np.array(param_list)
result = np.sum(param_list_np, axis=0)
result = result / len(param_list)
return result.tolist()
@staticmethod
def print_randoms():
ELib.__rnd_print_counter += 1
print('randoms [' + str(ELib.__rnd_print_counter) + '] > ',
'| python:' + str(random.random()),
'| numpy:' + str(np.random.random()),
'| torch:' + str(torch.rand(1, 1).item()))
@staticmethod
def averaged_tempered_softmax(np_arr,
temperature,
weights=None,
do_softmax=True):
if not isinstance(np_arr, np.ndarray):
np_arr = np.array(np_arr)
## calculate softmax over tables
dump = list()
if do_softmax:
for cur_pred in np_arr:
cur_pred_ts = torch.tensor(cur_pred)
if len(cur_pred.shape) == 1:
sft = F.softmax(cur_pred_ts / temperature,
dim=0).numpy().tolist()
else:
sft = F.softmax(cur_pred_ts / temperature, dim=1).numpy()
dump.append(sft)
else:
dump = np_arr
if len(np_arr.shape) == 2:
return dump
## prepare weights
if weights is None:
weights = [1.0 for _ in np_arr]
sum_weight = sum(weights)
## take average
result = np.array(dump[0]) * weights[0]
for ind in range(1, len(dump)):
result = result + dump[ind] * weights[ind]
result = result / sum_weight
result = result.tolist()
return result
@staticmethod
def calculate_metrics(truths, preds):
# sklearn turns on warnings after we import it, call this to make sute it is silent
warnings.filterwarnings('ignore')
if 0 <= min(truths) and max(truths) <= 1 and 0 <= min(preds) and max(
preds) <= 1:
F1 = metrics.f1_score(truths, preds)
Pre = metrics.precision_score(truths, preds)
Rec = metrics.recall_score(truths, preds)
else:
F1 = metrics.f1_score(truths, preds, average='weighted')
Pre = metrics.precision_score(truths, preds, average='weighted')
Rec = metrics.recall_score(truths, preds, average='weighted')
return [F1, Pre, Rec]
@staticmethod
def calculate_f1(truths, preds):
# sklearn turns on warnings after we import it, call this to make sute it is silent
warnings.filterwarnings('ignore')
if 0 <= min(truths) and max(truths) <= 1 and 0 <= min(preds) and max(
preds) <= 1:
F1 = metrics.f1_score(truths, preds)
else:
F1 = metrics.f1_score(truths, preds, average='weighted')
return F1
@staticmethod
def get_string_metrics(metrics):
return 'F1: {:.3f} Pre: {:.3f} Rec: {:.3f}'.format(
metrics[0], metrics[1], metrics[2])
@staticmethod
def majority_voting(lbl_list):
result = list()
for entry_ind, _ in enumerate(lbl_list[0]):
cur_lbls = [run[entry_ind] for run in lbl_list]
try:
cur_l = mode(cur_lbls)
except:
lbl_count = list()
for cur_lll in list(set(cur_lbls)):
lbl_count.append(
[cur_lll,
sum(run.count(cur_lll) for run in lbl_list)])
lbl_count.sort(key=lambda comp: comp[1])
cur_l = lbl_count[0][0]
result.append(cur_l)
return result
@staticmethod
def majority_logits(logit_list, weights=None):
probs = ELib.averaged_tempered_softmax(logit_list, 1, weights=weights)
result = list()
for cur_prob in probs:
lbl = cur_prob.index(max(cur_prob))
result.append(lbl)
return result
@staticmethod
def CrossEntropyLossWithSoftLabels(pred_tensor,
true_tensor,
temperature=1.0,
mean=True):
pred_softmaxed = F.log_softmax(pred_tensor / temperature, dim=1)
loss = -(true_tensor * pred_softmaxed).sum(dim=1)
if mean:
loss = loss.mean()
return loss
@staticmethod
def proxy_train(cls, *args, **kwargs):
try:
cls.train(*args, **kwargs)
except Exception as e:
print(
colored(
'Exception occurred in the training thread, model_id:{}>'.
format(cls.model_id), 'green'))
print(colored(traceback.format_exc(), 'red'))
print(
colored(
'__________________________________________________________',
'red'))
os._exit(1)
@staticmethod
def print_gpu_content(device):
if torch.cuda.is_available():
summary = torch.cuda.memory_summary(device)
print(
colored('----------------------------------------------',
'red'))
print(summary)
print(
colored('----------------------------------------------',
'red'))
ELib.PASS()
@staticmethod
def get_formatted_float_list(arg_list):
if type(arg_list[0]) is not list:
result = '[' + ','.join(
['{:.3f}'.format(entry) for entry in arg_list]) + ']'
else:
result = '[\n'
for cur_row in arg_list:
result += ' [' + ','.join(
['{:.3f}'.format(entry) for entry in cur_row]) + ']\n'
result += ']'
return result
@staticmethod
def save_histogram(title, series_list, label_list, bins, output_dir,
file_name):
fig, ax = plt.subplots()
ax.hist(series_list, label=label_list, bins=bins)
ax.legend(fontsize=10, loc='best')
plt.title(title, fontsize=20)
plt.savefig(output_dir + file_name) # save should be before show() !
# plt.show()
ELib.PASS()
@staticmethod
def one_hot(lbl_list, cls_count):
result = np.eye(cls_count)[lbl_list]
return result.astype(np.int)
@staticmethod
def logit_to_label(logit_list):
result = np.argmax(logit_list, axis=1).tolist()
return result
@staticmethod
def logit_to_prob(logit_list):
result = F.softmax(torch.tensor(logit_list), dim=1).numpy().tolist()
return result
def support(request: HttpRequest) -> HttpResponse:
context: Dict[str, Any] = {}
if "success_message" in request.session:
context["success_message"] = request.session["success_message"]
del request.session["success_message"]
if settings.BILLING_ENABLED and request.method == "POST":
# We check that request.POST only has two keys in it: The
# realm_id and a field to change.
keys = set(request.POST.keys())
if "csrfmiddlewaretoken" in keys:
keys.remove("csrfmiddlewaretoken")
if len(keys) != 2:
raise JsonableError(_("Invalid parameters"))
realm_id = request.POST.get("realm_id")
realm = Realm.objects.get(id=realm_id)
if request.POST.get("plan_type", None) is not None:
new_plan_type = int(request.POST.get("plan_type"))
current_plan_type = realm.plan_type
do_change_plan_type(realm, new_plan_type, acting_user=request.user)
msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(new_plan_type)} "
context["success_message"] = msg
elif request.POST.get("discount", None) is not None:
new_discount = Decimal(request.POST.get("discount"))
current_discount = get_discount_for_realm(realm) or 0
attach_discount_to_realm(realm,
new_discount,
acting_user=request.user)
context[
"success_message"] = f"Discount of {realm.string_id} changed to {new_discount}% from {current_discount}%."
elif request.POST.get("new_subdomain", None) is not None:
new_subdomain = request.POST.get("new_subdomain")
old_subdomain = realm.string_id
try:
check_subdomain_available(new_subdomain)
except ValidationError as error:
context["error_message"] = error.message
else:
do_change_realm_subdomain(realm,
new_subdomain,
acting_user=request.user)
request.session[
"success_message"] = f"Subdomain changed from {old_subdomain} to {new_subdomain}"
return HttpResponseRedirect(
reverse("support") + "?" + urlencode({"q": new_subdomain}))
elif request.POST.get("status", None) is not None:
status = request.POST.get("status")
if status == "active":
do_send_realm_reactivation_email(realm,
acting_user=request.user)
context[
"success_message"] = f"Realm reactivation email sent to admins of {realm.string_id}."
elif status == "deactivated":
do_deactivate_realm(realm, acting_user=request.user)
context["success_message"] = f"{realm.string_id} deactivated."
elif request.POST.get("billing_method", None) is not None:
billing_method = request.POST.get("billing_method")
if billing_method == "send_invoice":
update_billing_method_of_current_plan(
realm,
charge_automatically=False,
acting_user=request.user)
context[
"success_message"] = f"Billing method of {realm.string_id} updated to pay by invoice."
elif billing_method == "charge_automatically":
update_billing_method_of_current_plan(
realm, charge_automatically=True, acting_user=request.user)
context[
"success_message"] = f"Billing method of {realm.string_id} updated to charge automatically."
elif request.POST.get("sponsorship_pending", None) is not None:
sponsorship_pending = request.POST.get("sponsorship_pending")
if sponsorship_pending == "true":
update_sponsorship_status(realm,
True,
acting_user=request.user)
context[
"success_message"] = f"{realm.string_id} marked as pending sponsorship."
elif sponsorship_pending == "false":
update_sponsorship_status(realm,
False,
acting_user=request.user)
context[
"success_message"] = f"{realm.string_id} is no longer pending sponsorship."
elif request.POST.get("approve_sponsorship") is not None:
if request.POST.get(
"approve_sponsorship") == "approve_sponsorship":
approve_sponsorship(realm, acting_user=request.user)
context[
"success_message"] = f"Sponsorship approved for {realm.string_id}"
elif request.POST.get("downgrade_method", None) is not None:
downgrade_method = request.POST.get("downgrade_method")
if downgrade_method == "downgrade_at_billing_cycle_end":
downgrade_at_the_end_of_billing_cycle(realm)
context[
"success_message"] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
elif downgrade_method == "downgrade_now_without_additional_licenses":
downgrade_now_without_creating_additional_invoices(realm)
context[
"success_message"] = f"{realm.string_id} downgraded without creating additional invoices"
elif downgrade_method == "downgrade_now_void_open_invoices":
downgrade_now_without_creating_additional_invoices(realm)
voided_invoices_count = void_all_open_invoices(realm)
context[
"success_message"] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
elif request.POST.get("scrub_realm", None) is not None:
if request.POST.get("scrub_realm") == "scrub_realm":
do_scrub_realm(realm, acting_user=request.user)
context["success_message"] = f"{realm.string_id} scrubbed."
query = request.GET.get("q", None)
if query:
key_words = get_invitee_emails_set(query)
users = set(UserProfile.objects.filter(delivery_email__in=key_words))
realms = set(Realm.objects.filter(string_id__in=key_words))
for key_word in key_words:
try:
URLValidator()(key_word)
parse_result = urllib.parse.urlparse(key_word)
hostname = parse_result.hostname
assert hostname is not None
if parse_result.port:
hostname = f"{hostname}:{parse_result.port}"
subdomain = get_subdomain_from_hostname(hostname)
try:
realms.add(get_realm(subdomain))
except Realm.DoesNotExist:
pass
except ValidationError:
users.update(
UserProfile.objects.filter(full_name__iexact=key_word))
for realm in realms:
realm.customer = get_customer_by_realm(realm)
current_plan = get_current_plan_by_realm(realm)
if current_plan is not None:
new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(
current_plan, timezone_now())
if last_ledger_entry is not None:
if new_plan is not None:
realm.current_plan = new_plan
else:
realm.current_plan = current_plan
realm.current_plan.licenses = last_ledger_entry.licenses
realm.current_plan.licenses_used = get_latest_seat_count(
realm)
# full_names can have , in them
users.update(UserProfile.objects.filter(full_name__iexact=query))
context["users"] = users
context["realms"] = realms
confirmations: List[Dict[str, Any]] = []
preregistration_users = PreregistrationUser.objects.filter(
email__in=key_words)
confirmations += get_confirmations(
[
Confirmation.USER_REGISTRATION, Confirmation.INVITATION,
Confirmation.REALM_CREATION
],
preregistration_users,
hostname=request.get_host(),
)
multiuse_invites = MultiuseInvite.objects.filter(realm__in=realms)
confirmations += get_confirmations([Confirmation.MULTIUSE_INVITE],
multiuse_invites)
confirmations += get_confirmations([Confirmation.REALM_REACTIVATION],
[realm.id for realm in realms])
context["confirmations"] = confirmations
def get_realm_owner_emails_as_string(realm: Realm) -> str:
return ", ".join(realm.get_human_owner_users().order_by(
"delivery_email").values_list("delivery_email", flat=True))
def get_realm_admin_emails_as_string(realm: Realm) -> str:
return ", ".join(
realm.get_human_admin_users(include_realm_owners=False).order_by(
"delivery_email").values_list("delivery_email", flat=True))
context[
"get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string
context[
"get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string
context["get_discount_for_realm"] = get_discount_for_realm
context["get_org_type_display_name"] = get_org_type_display_name
context["realm_icon_url"] = realm_icon_url
context["Confirmation"] = Confirmation
return render(request, "analytics/support.html", context=context)
def validator(self):
return URLValidator()
class InvenTreeSetting(models.Model):
"""
An InvenTreeSetting object is a key:value pair used for storing
single values (e.g. one-off settings values).
The class provides a way of retrieving the value for a particular key,
even if that key does not exist.
"""
"""
Dict of all global settings values:
The key of each item is the name of the value as it appears in the database.
Each global setting has the following parameters:
- name: Translatable string name of the setting (required)
- description: Translatable string description of the setting (required)
- default: Default value (optional)
- units: Units of the particular setting (optional)
- validator: Validation function for the setting (optional)
The keys must be upper-case
"""
GLOBAL_SETTINGS = {
'INVENTREE_INSTANCE': {
'name': _('InvenTree Instance Name'),
'default': 'InvenTree server',
'description': _('String descriptor for the server instance'),
},
'INVENTREE_COMPANY_NAME': {
'name': _('Company name'),
'description': _('Internal company name'),
'default': 'My company name',
},
'INVENTREE_BASE_URL': {
'name': _('Base URL'),
'description': _('Base URL for server instance'),
'validator': URLValidator(),
'default': '',
},
'INVENTREE_DEFAULT_CURRENCY': {
'name': _('Default Currency'),
'description': _('Default currency'),
'default': 'USD',
'choices': djmoney.settings.CURRENCY_CHOICES,
},
'INVENTREE_DOWNLOAD_FROM_URL': {
'name':
_('Download from URL'),
'description':
_('Allow download of remote images and files from external URL'),
'validator':
bool,
'default':
False,
},
'BARCODE_ENABLE': {
'name': _('Barcode Support'),
'description': _('Enable barcode scanner support'),
'default': True,
'validator': bool,
},
'PART_IPN_REGEX': {
'name': _('IPN Regex'),
'description':
_('Regular expression pattern for matching Part IPN')
},
'PART_ALLOW_DUPLICATE_IPN': {
'name': _('Allow Duplicate IPN'),
'description': _('Allow multiple parts to share the same IPN'),
'default': True,
'validator': bool,
},
'PART_ALLOW_EDIT_IPN': {
'name': _('Allow Editing IPN'),
'description':
_('Allow changing the IPN value while editing a part'),
'default': True,
'validator': bool,
},
'PART_COPY_BOM': {
'name': _('Copy Part BOM Data'),
'description':
_('Copy BOM data by default when duplicating a part'),
'default': True,
'validator': bool,
},
'PART_COPY_PARAMETERS': {
'name':
_('Copy Part Parameter Data'),
'description':
_('Copy parameter data by default when duplicating a part'),
'default':
True,
'validator':
bool,
},
'PART_COPY_TESTS': {
'name': _('Copy Part Test Data'),
'description':
_('Copy test data by default when duplicating a part'),
'default': True,
'validator': bool
},
'PART_CATEGORY_PARAMETERS': {
'name':
_('Copy Category Parameter Templates'),
'description':
_('Copy category parameter templates when creating a part'),
'default':
True,
'validator':
bool
},
'PART_RECENT_COUNT': {
'name': _('Recent Part Count'),
'description':
_('Number of recent parts to display on index page'),
'default': 10,
'validator': [int, MinValueValidator(1)]
},
'PART_TEMPLATE': {
'name': _('Template'),
'description': _('Parts are templates by default'),
'default': False,
'validator': bool,
},
'PART_ASSEMBLY': {
'name':
_('Assembly'),
'description':
_('Parts can be assembled from other components by default'),
'default':
False,
'validator':
bool,
},
'PART_COMPONENT': {
'name': _('Component'),
'description': _('Parts can be used as sub-components by default'),
'default': True,
'validator': bool,
},
'PART_PURCHASEABLE': {
'name': _('Purchaseable'),
'description': _('Parts are purchaseable by default'),
'default': False,
'validator': bool,
},
'PART_SALABLE': {
'name': _('Salable'),
'description': _('Parts are salable by default'),
'default': False,
'validator': bool,
},
'PART_TRACKABLE': {
'name': _('Trackable'),
'description': _('Parts are trackable by default'),
'default': False,
'validator': bool,
},
'PART_VIRTUAL': {
'name': _('Virtual'),
'description': _('Parts are virtual by default'),
'default': False,
'validator': bool,
},
'PART_SHOW_QUANTITY_IN_FORMS': {
'name': _('Show Quantity in Forms'),
'description': _('Display available part quantity in some forms'),
'default': True,
'validator': bool,
},
'REPORT_DEBUG_MODE': {
'name': _('Debug Mode'),
'description': _('Generate reports in debug mode (HTML output)'),
'default': False,
'validator': bool,
},
'REPORT_DEFAULT_PAGE_SIZE': {
'name': _('Page Size'),
'description': _('Default page size for PDF reports'),
'default': 'A4',
'choices': [('A4', 'A4'), ('Legal', 'Legal'),
('Letter', 'Letter')],
},
'REPORT_ENABLE_TEST_REPORT': {
'name': _('Test Reports'),
'description': _('Enable generation of test reports'),
'default': True,
'validator': bool,
},
'STOCK_ENABLE_EXPIRY': {
'name': _('Stock Expiry'),
'description': _('Enable stock expiry functionality'),
'default': False,
'validator': bool,
},
'STOCK_ALLOW_EXPIRED_SALE': {
'name': _('Sell Expired Stock'),
'description': _('Allow sale of expired stock'),
'default': False,
'validator': bool,
},
'STOCK_STALE_DAYS': {
'name':
_('Stock Stale Time'),
'description':
_('Number of days stock items are considered stale before expiring'
),
'default':
0,
'units':
_('days'),
'validator': [int],
},
'STOCK_ALLOW_EXPIRED_BUILD': {
'name': _('Build Expired Stock'),
'description': _('Allow building with expired stock'),
'default': False,
'validator': bool,
},
'STOCK_OWNERSHIP_CONTROL': {
'name':
_('Stock Ownership Control'),
'description':
_('Enable ownership control over stock locations and items'),
'default':
False,
'validator':
bool,
},
'STOCK_GROUP_BY_PART': {
'name': _('Group by Part'),
'description':
_('Group stock items by part reference in table views'),
'default': True,
'validator': bool,
},
'STOCK_RECENT_COUNT': {
'name':
_('Recent Stock Count'),
'description':
_('Number of recent stock items to display on index page'),
'default':
10,
'validator': [int, MinValueValidator(1)]
},
'BUILDORDER_REFERENCE_PREFIX': {
'name': _('Build Order Reference Prefix'),
'description': _('Prefix value for build order reference'),
'default': 'BO',
},
'BUILDORDER_REFERENCE_REGEX': {
'name':
_('Build Order Reference Regex'),
'description':
_('Regular expression pattern for matching build order reference')
},
'SALESORDER_REFERENCE_PREFIX': {
'name': _('Sales Order Reference Prefix'),
'description': _('Prefix value for sales order reference'),
'default': 'SO',
},
'PURCHASEORDER_REFERENCE_PREFIX': {
'name': _('Purchase Order Reference Prefix'),
'description': _('Prefix value for purchase order reference'),
'default': 'PO',
},
}
class Meta:
verbose_name = "InvenTree Setting"
verbose_name_plural = "InvenTree Settings"
@classmethod
def get_setting_name(cls, key):
"""
Return the name of a particular setting.
If it does not exist, return an empty string.
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
return setting.get('name', '')
else:
return ''
@classmethod
def get_setting_description(cls, key):
"""
Return the description for a particular setting.
If it does not exist, return an empty string.
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
return setting.get('description', '')
else:
return ''
@classmethod
def get_setting_units(cls, key):
"""
Return the units for a particular setting.
If it does not exist, return an empty string.
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
return setting.get('units', '')
else:
return ''
@classmethod
def get_setting_validator(cls, key):
"""
Return the validator for a particular setting.
If it does not exist, return None
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
return setting.get('validator', None)
else:
return None
@classmethod
def get_setting_default(cls, key):
"""
Return the default value for a particular setting.
If it does not exist, return an empty string
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
return setting.get('default', '')
else:
return ''
@classmethod
def get_setting_choices(cls, key):
"""
Return the validator choices available for a particular setting.
"""
key = str(key).strip().upper()
if key in cls.GLOBAL_SETTINGS:
setting = cls.GLOBAL_SETTINGS[key]
choices = setting.get('choices', None)
else:
choices = None
"""
TODO:
if type(choices) is function:
# Evaluate the function (we expect it will return a list of tuples...)
return choices()
"""
return choices
@classmethod
def get_setting_object(cls, key):
"""
Return an InvenTreeSetting object matching the given key.
- Key is case-insensitive
- Returns None if no match is made
"""
key = str(key).strip().upper()
try:
setting = InvenTreeSetting.objects.filter(key__iexact=key).first()
except (ValueError, InvenTreeSetting.DoesNotExist):
setting = None
except (IntegrityError, OperationalError):
setting = None
# Setting does not exist! (Try to create it)
if not setting:
setting = InvenTreeSetting(
key=key, value=InvenTreeSetting.get_setting_default(key))
try:
# Wrap this statement in "atomic", so it can be rolled back if it fails
with transaction.atomic():
setting.save()
except (IntegrityError, OperationalError):
# It might be the case that the database isn't created yet
pass
return setting
@classmethod
def get_setting_pk(cls, key):
"""
Return the primary-key value for a given setting.
If the setting does not exist, return None
"""
setting = InvenTreeSetting.get_setting_object(cls)
if setting:
return setting.pk
else:
return None
@classmethod
def get_setting(cls, key, backup_value=None):
"""
Get the value of a particular setting.
If it does not exist, return the backup value (default = None)
"""
# If no backup value is specified, atttempt to retrieve a "default" value
if backup_value is None:
backup_value = cls.get_setting_default(key)
setting = InvenTreeSetting.get_setting_object(key)
if setting:
value = setting.value
# If the particular setting is defined as a boolean, cast the value to a boolean
if setting.is_bool():
value = InvenTree.helpers.str2bool(value)
if setting.is_int():
try:
value = int(value)
except (ValueError, TypeError):
value = backup_value
else:
value = backup_value
return value
@classmethod
def set_setting(cls, key, value, user, create=True):
"""
Set the value of a particular setting.
If it does not exist, option to create it.
Args:
key: settings key
value: New value
user: User object (must be staff member to update a core setting)
create: If True, create a new setting if the specified key does not exist.
"""
if user is not None and not user.is_staff:
return
try:
setting = InvenTreeSetting.objects.get(key__iexact=key)
except InvenTreeSetting.DoesNotExist:
if create:
setting = InvenTreeSetting(key=key)
else:
return
# Enforce standard boolean representation
if setting.is_bool():
value = InvenTree.helpers.str2bool(value)
setting.value = str(value)
setting.save()
key = models.CharField(
max_length=50,
blank=False,
unique=True,
help_text=_('Settings key (must be unique - case insensitive'))
value = models.CharField(max_length=200,
blank=True,
unique=False,
help_text=_('Settings value'))
@property
def name(self):
return InvenTreeSetting.get_setting_name(self.key)
@property
def default_value(self):
return InvenTreeSetting.get_setting_default(self.key)
@property
def description(self):
return InvenTreeSetting.get_setting_description(self.key)
@property
def units(self):
return InvenTreeSetting.get_setting_units(self.key)
def clean(self):
"""
If a validator (or multiple validators) are defined for a particular setting key,
run them against the 'value' field.
"""
super().clean()
validator = InvenTreeSetting.get_setting_validator(self.key)
if self.is_bool():
self.value = InvenTree.helpers.str2bool(self.value)
if self.is_int():
try:
self.value = int(self.value)
except (ValueError):
raise ValidationError(_('Must be an integer value'))
if validator is not None:
self.run_validator(validator)
def run_validator(self, validator):
"""
Run a validator against the 'value' field for this InvenTreeSetting object.
"""
if validator is None:
return
value = self.value
# Boolean validator
if self.is_bool():
# Value must "look like" a boolean value
if InvenTree.helpers.is_bool(value):
# Coerce into either "True" or "False"
value = InvenTree.helpers.str2bool(value)
else:
raise ValidationError(
{'value': _('Value must be a boolean value')})
# Integer validator
if self.is_int():
try:
# Coerce into an integer value
value = int(value)
except (ValueError, TypeError):
raise ValidationError({
'value':
_('Value must be an integer value'),
})
# If a list of validators is supplied, iterate through each one
if type(validator) in [list, tuple]:
for v in validator:
self.run_validator(v)
if callable(validator):
# We can accept function validators with a single argument
validator(self.value)
def validate_unique(self, exclude=None):
""" Ensure that the key:value pair is unique.
In addition to the base validators, this ensures that the 'key'
is unique, using a case-insensitive comparison.
"""
super().validate_unique(exclude)
try:
setting = InvenTreeSetting.objects.exclude(id=self.id).filter(
key__iexact=self.key)
if setting.exists():
raise ValidationError({'key': _('Key string must be unique')})
except InvenTreeSetting.DoesNotExist:
pass
def choices(self):
"""
Return the available choices for this setting (or None if no choices are defined)
"""
return InvenTreeSetting.get_setting_choices(self.key)
def is_bool(self):
"""
Check if this setting is required to be a boolean value
"""
validator = InvenTreeSetting.get_setting_validator(self.key)
if validator == bool:
return True
if type(validator) in [list, tuple]:
for v in validator:
if v == bool:
return True
def as_bool(self):
"""
Return the value of this setting converted to a boolean value.
Warning: Only use on values where is_bool evaluates to true!
"""
return InvenTree.helpers.str2bool(self.value)
def is_int(self):
"""
Check if the setting is required to be an integer value:
"""
validator = InvenTreeSetting.get_setting_validator(self.key)
if validator == int:
return True
if type(validator) in [list, tuple]:
for v in validator:
if v == int:
return True
return False
def as_int(self):
"""
Return the value of this setting converted to a boolean value.
If an error occurs, return the default value
"""
try:
value = int(self.value)
except (ValueError, TypeError):
value = self.default_value()
return value
def validate_url(value):
url_validator = URLValidator()
try:
url_validator(value)
except:
raise ValidationError("Please provide valid URL")
def replace_from_dict(self, data):
"""Takes the given dictionary and modifies this survey version and its
associated questions. Uses the same format as
:func:`SurveyVersion.to_dict` with one additional (optional) key
"remove" which contains a list of :class:`Question` ids to be removed
from the ``Survey``.
:param data:
Dictionary to overwrite the contents of the ``Survey`` and
associated :class:`Question` objects with.
:raises EditNotAllowedException:
If the version being replaced is active
:raises Question.DoesNotExist:
If a question id is referenced that does not exist or is not
associated with this survey.
:raises ValidationError:
If the name or success_redirect URL are blank or if the URL is
invalid
"""
self.validate_editable()
errors = {}
name = data.get('name', '').strip()
url = data.get('redirect_url', '').strip()
if not name:
errors['name'] = 'name cannot be blank'
else:
self.survey.name = name
if not url:
errors['redirect_url'] = 'redirect URL cannot be blank'
else:
validator = URLValidator(schemes=['http', 'https'])
try:
validator(url)
self.survey.success_redirect = url
except ValidationError:
errors['redirect_url'] = ('invalid URL; only fully qualified '
'URLs are supported')
show_title = data.get('show_title', False)
self.survey.show_title = show_title
if errors:
raise ValidationError('Survey Validation Failed', params=errors)
else:
self.survey.save()
if 'questions' in data:
for q_data in data['questions']:
if q_data['id'] == 0:
# new question
kwargs = {
'field': FIELDS_DICT[q_data['field_key']],
'text': q_data['text'],
'required': q_data['required'],
'field_parms': q_data['field_parms'],
}
# add the question and set the data's question id so that we
# can do re-ordering down below
question = self.add_question(**kwargs)
q_data['id'] = question.id
else:
question = Question.objects.get(
id=q_data['id'], survey_versions__id=self.id)
question.text = q_data['text']
question.required = q_data['required']
question.field_parms = q_data['field_parms']
question.save()
# fix the ranking -- creating new questions will mess with the order
if 'questions' in data:
for index, q_data in enumerate(data['questions']):
question = Question.objects.get(id=q_data['id'],
survey_versions__id=self.id)
q_order = QuestionOrder.objects.get(question=question,
survey_version=self)
q_order.rank = index + 1
q_order.save(rerank=False)
if 'remove' in data:
for id in data['remove']:
question = Question.objects.get(id=id,
survey_versions__id=self.id)
self.remove_question(question)
def validate(self, data):
port_re = "(:[0-9]{1,5}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}" \
"|655[0-2][0-9]|6553[0-5])"
if not self.context['request'].method == 'PATCH':
if ('host' not in data or 'protocol' not in data
or 'path' not in data or 'query' not in data
or 'fragment' not in data):
raise serializers.ValidationError(
'Please provide valid host, protocol, path, query and '
'fragment')
protocol = data['protocol']
path = data['path']
query = data['query']
fragment = data['fragment']
host = data['host']
else:
protocol = data.get('protocol', self.instance.protocol)
path = data.get('path', self.instance.path)
query = data.get('query', self.instance.query)
fragment = data.get('fragment', self.instance.fragment)
host = data.get('host', self.instance.host)
product = data.get('product', None)
from urlparse import urlunsplit
if protocol:
endpoint = urlunsplit((protocol, host, path, query, fragment))
else:
endpoint = host
from django.core import exceptions
from django.core.validators import RegexValidator
import re
try:
url_validator = URLValidator()
url_validator(endpoint)
except exceptions.ValidationError:
try:
# do we have a port number?
regex = re.compile(port_re)
host = endpoint
if regex.findall(endpoint):
for g in regex.findall(endpoint):
host = re.sub(port_re, '', host)
validate_ipv46_address(host)
except exceptions.ValidationError:
try:
validate_hostname = RegexValidator(
regex=r'[a-zA-Z0-9-_]*\.[a-zA-Z]{2,6}')
# do we have a port number?
regex = re.compile(port_re)
host = endpoint
if regex.findall(endpoint):
for g in regex.findall(endpoint):
host = re.sub(port_re, '', host)
validate_hostname(host)
except: # noqa
raise serializers.ValidationError(
'It does not appear as though this endpoint is a '
'valid URL or IP address.',
code='invalid')
endpoint = Endpoint.objects.filter(protocol=protocol,
host=host,
path=path,
query=query,
fragment=fragment,
product=product)
if endpoint.count() > 0 and not self.instance:
raise serializers.ValidationError(
'It appears as though an endpoint with this data already '
'exists for this product.',
code='invalid')
return data
(MaxValueValidator(0), 1, ValidationError), (MaxValueValidator(NOW), NOW + timedelta(days=1), ValidationError), (MinValueValidator(-10), -10, None), (MinValueValidator(-10), 10, None), (MinValueValidator(-10), 0, None), (MinValueValidator(NOW), NOW, None), (MinValueValidator(NOW), NOW + timedelta(days=1), None), (MinValueValidator(0), -1, ValidationError), (MinValueValidator(NOW), NOW - timedelta(days=1), ValidationError), (MaxLengthValidator(10), '', None), (MaxLengthValidator(10), 10 * 'x', None), (MaxLengthValidator(10), 15 * 'x', ValidationError), (MinLengthValidator(10), 15 * 'x', None), (MinLengthValidator(10), 10 * 'x', None), (MinLengthValidator(10), '', ValidationError), (URLValidator(EXTENDED_SCHEMES), 'file://localhost/path', None), (URLValidator(EXTENDED_SCHEMES), 'git://example.com/', None), (URLValidator(EXTENDED_SCHEMES), 'git+ssh://[email protected]/example/hg-git.git', None), (URLValidator(EXTENDED_SCHEMES), 'git://-invalid.com', ValidationError), # Trailing newlines not accepted (URLValidator(), 'http://www.djangoproject.com/\n', ValidationError), (URLValidator(), 'http://[::ffff:192.9.5.5]\n', ValidationError), # Trailing junk does not take forever to reject (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br ', ValidationError), (URLValidator(), 'http://www.asdasdasdasdsadfm.com.br z', ValidationError), (BaseValidator(True), True, None), (BaseValidator(True), False, ValidationError), (RegexValidator(), '', None), (RegexValidator(), 'x1x2', None), (RegexValidator('[0-9]+'), 'xxxxxx', ValidationError),
'Y-m-d H:i')
SHORT_TIME_FORMAT = getattr(configuration, 'SHORT_TIME_FORMAT', 'H:i:s')
STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None)
STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {})
TIME_FORMAT = getattr(configuration, 'TIME_FORMAT', 'g:i a')
TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC')
# Check for hard-coded dynamic config parameters
for param in PARAMS:
if hasattr(configuration, param.name):
globals()[param.name] = getattr(configuration, param.name)
# Validate update repo URL and timeout
if RELEASE_CHECK_URL:
validator = URLValidator(
message=("RELEASE_CHECK_URL must be a valid API URL. Example: "
"https://api.github.com/repos/netbox-community/netbox"))
try:
validator(RELEASE_CHECK_URL)
except ValidationError as err:
raise ImproperlyConfigured(str(err))
#
# Database
#
# Only PostgreSQL is supported
if METRICS_ENABLED:
DATABASE.update({'ENGINE': 'django_prometheus.db.backends.postgresql'})
else:
DATABASE.update({'ENGINE': 'django.db.backends.postgresql'})
def clean(self):
if self.link_is_view_name:
view_name_validator(self.link)
else:
URLValidator(schemes=('http', 'https'))(self.link)
class RemoteRepository(models.Model):
"""
Remote importable repositories.
This models Github and Bitbucket importable repositories
"""
# Auto fields
pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True)
modified_date = models.DateTimeField(_('Modified date'), auto_now=True)
# This should now be a OneToOne
users = models.ManyToManyField(
User,
verbose_name=_('Users'),
related_name='oauth_repositories',
)
account = models.ForeignKey(
SocialAccount,
verbose_name=_('Connected account'),
related_name='remote_repositories',
null=True,
blank=True,
on_delete=models.CASCADE,
)
organization = models.ForeignKey(
RemoteOrganization,
verbose_name=_('Organization'),
related_name='repositories',
null=True,
blank=True,
on_delete=models.CASCADE,
)
active = models.BooleanField(_('Active'), default=False)
project = models.OneToOneField(
Project,
on_delete=models.SET_NULL,
related_name='remote_repository',
null=True,
blank=True,
)
name = models.CharField(_('Name'), max_length=255)
full_name = models.CharField(
_('Full Name'),
max_length=255,
db_index=True,
)
description = models.TextField(
_('Description'),
blank=True,
null=True,
help_text=_('Description of the project'),
)
avatar_url = models.URLField(
_('Owner avatar image URL'),
null=True,
blank=True,
)
ssh_url = models.URLField(
_('SSH URL'),
max_length=512,
blank=True,
validators=[URLValidator(schemes=['ssh'])],
)
clone_url = models.URLField(
_('Repository clone URL'),
max_length=512,
blank=True,
validators=[
URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn']),
],
)
html_url = models.URLField(_('HTML URL'), null=True, blank=True)
private = models.BooleanField(_('Private repository'), default=False)
admin = models.BooleanField(_('Has admin privilege'), default=False)
vcs = models.CharField(
_('vcs'),
max_length=200,
blank=True,
choices=REPO_CHOICES,
)
json = models.TextField(_('Serialized API response'))
objects = RemoteRepositoryQuerySet.as_manager()
class Meta:
ordering = ['organization__name', 'name']
verbose_name_plural = 'remote repositories'
def __str__(self):
return 'Remote repository: {}'.format(self.html_url)
def get_serialized(self, key=None, default=None):
try:
data = json.loads(self.json)
if key is not None:
return data.get(key, default)
return data
except ValueError:
pass
@property
def clone_fuzzy_url(self):
"""Try to match against several permutations of project URL."""
def matches(self, user):
"""Projects that exist with repository URL already."""
# Support Git scheme GitHub url format that may exist in database
truncated_url = self.clone_url.replace('.git', '')
http_url = self.clone_url.replace('git://',
'https://').replace('.git', '')
projects = Project.objects.public(user).filter(
Q(repo=self.clone_url) | Q(repo=truncated_url)
| Q(repo=truncated_url + '.git') | Q(repo=http_url)
| Q(repo=http_url + '.git')).values('slug')
return [{
'id':
project['slug'],
'url':
reverse(
'projects_detail',
kwargs={
'project_slug': project['slug'],
},
),
} for project in projects]
class URLField(forms.URLField):
"""
Overloading default URLField to accept only http and https schemes
"""
default_validators = [URLValidator(schemes=['http', 'https'])]
class FollowFeedForm(forms.Form):
url = forms.CharField(label="Feed URL",
max_length=200,
validators=[URLValidator()])
def clean(self):
number_message = "Enter a number."
scale_preset = self.cleaned_data.get('scale_preset', '')
if scale_preset == '5':
# custom preset error handling
scale_type = self.cleaned_data.get('scale_type', '')
if scale_type == 'ul':
scale_lower = self.cleaned_data.get('scale_lower')
scale_upper = self.cleaned_data.get('scale_upper')
if not scale_lower:
self._errors['scale_lower'] = self.error_class(
[number_message])
if not scale_upper:
self._errors['scale_upper'] = self.error_class(
[number_message])
elif scale_type == 'ev':
scale_est = self.cleaned_data.get('scale_est')
scale_err = self.cleaned_data.get('scale_err')
if not scale_est:
self._errors['scale_est'] = self.error_class(
[number_message])
if not scale_err:
self._errors['scale_err'] = self.error_class(
[number_message])
else:
# if scale isn't custom, use preset settings
self.cleaned_data['scale_type'] = 'ul'
self.cleaned_data['scale_units'] = 'degwidth'
self.cleaned_data['scale_lower'] = self.SCALE_PRESET_SETTINGS[
scale_preset][0]
self.cleaned_data['scale_upper'] = self.SCALE_PRESET_SETTINGS[
scale_preset][1]
center_ra = self.cleaned_data.get('center_ra')
center_dec = self.cleaned_data.get('center_dec')
radius = self.cleaned_data.get('radius')
if center_ra or center_dec or radius:
if not center_ra:
self._errors['center_ra'] = self.error_class([number_message])
if not center_dec:
self._errors['center_dec'] = self.error_class([number_message])
if not radius:
self._errors['radius'] = self.error_class([number_message])
tweak_order = self.cleaned_data.get('tweak_order')
if tweak_order < 0 or tweak_order > 9:
self._errors['tweak_order'] = self.error_class(
['Tweak order must be between 0 and 9'])
upload_type = self.cleaned_data.get('upload_type', '')
if upload_type == 'file':
if not self.cleaned_data.get('file'):
raise forms.ValidationError(
"You must select a file to upload.")
elif upload_type == 'url':
url = self.cleaned_data.get('url', '')
if not (url.startswith('http://') or url.startswith('ftp://')
or url.startswith('https://')):
url = 'http://' + url
if url.startswith('http://http://') or url.startswith(
'http://ftp://') or url.startswith('http://https://'):
url = url[7:]
if len(url) == 0:
raise forms.ValidationError("You must enter a url to upload.")
print('Cleaned URL:', url)
urlvalidator = URLValidator()
try:
urlvalidator(url)
except forms.ValidationError:
raise forms.ValidationError("You must enter a valid url.")
self.cleaned_data['url'] = url
return self.cleaned_data
async def on_message(message):
msg = word(message.content)
val = URLValidator()
if message.author.id in admins:
if msg.startswith(prefix + 'add '):
msg.clearfront(len(prefix) + 4)
subject, link, version = None, None, None
try:
subject, link, version = msg.word.split(', ', 3)
subject = subject.lower()
except ValueError:
await message.channel.send(
f'```Error! Invalid Syntax! Correct syntax: {prefix}add <subject>, <link>, <version>```'
)
return
else:
try:
val(link)
except ValidationError:
await message.channel.send(
f'```Error! Invalid Link! Make sure your link includes https/http```'
)
return
else:
if subject in alias.keys():
subject = alias[subject]
elif subject not in alias.values():
await message.channel.send(
f'```Invalid subject \"{subject}\"!```')
return
now = datetime.now(tz=pytz.timezone('Asia/Singapore'))
month = '%02d' % now.month
day = '%02d' % now.day
hour = '%02d' % now.hour
minute = '%02d' % now.minute
second = '%02d' % now.second
if subject in links.keys():
links[subject][version] = [
link, f'{now.year}/{month}/{day} {hour}:{minute}:{second}'
]
else:
links[subject] = {
version: [
link,
f'{now.year}/{month}/{day} {hour}:{minute}:{second}'
]
}
await message.channel.send(embed=createEmbed(
title='Success',
text=f'You have uploaded {version} of {subject} ( {link} )',
colour=0x00ff7f))
await client.get_user(611513933710229504).send(
f'<@!611513933710229504>, <@!{message.author.id}> has uploaded {version} of {subject} at {link}.'
)
elif msg.startswith(prefix + 'del '):
msg.clearfront(len(prefix) + 4)
subject, version = None, None
try:
subject, version = msg.word.split(', ', 2)
subject = subject.lower()
except ValueError:
await message.channel.send(
f'```Error! Invalid Syntax! Correct syntax: {prefix}del <subject>, <version>```'
)
else:
if subject in alias.keys():
subject = alias[subject]
elif subject not in alias.values():
await message.channel.send(
f'```Invalid subject \"{subject}\"!```')
return
if version in links[subject].keys():
await message.channel.send(embed=createEmbed(
title='Success',
text=
f'You deleted {version} of {subject} (Link: {links[subject][version][0]}, Uploaded: {links[subject][version][1]})!',
colour=0x00ff7f))
await client.get_user(611513933710229504).send(
f'<@!611513933710229504>, <@!{message.author.id}> has deleted {version} of {subject} at {links[subject][version][0]} [{links[subject][version][1]}].'
)
links[subject].pop(version)
else:
await message.channel.send(
f'```Error! Invalid Syntax! Correct syntax: {prefix}del <subject>, <version>```'
)
if msg.startswith(prefix + 'list '):
msg.clearfront(len(prefix) + 5)
subject = msg.word.lower()
if subject in alias.keys():
subject = alias[subject]
elif subject not in alias.values():
await message.channel.send(f'Invalid subject \"{subject}\"!')
return
txt = ''
if subject not in links.keys():
await message.channel.send(
f'```No data found for subject \"{subject}\"!```')
return
x = len(links[subject])
for version, link in links[subject].items():
if x == 1:
txt += f'\n\n**Latest Version**:\n{version} - {link[0]} [{link[1]}]'
else:
txt += f'{version} - {link[0]} [{link[1]}]\n'
x -= 1
if txt != '':
await message.channel.send(embed=createEmbed(
title=f'Versions of {subject} notes',
text=txt,
colour=discord.Color.blue(),
footer=
'Press Ctrl + P to see the document(and then you can access the bookmarks)'
))
else:
await message.channel.send(
f'```No data found for subject \"{subject}\"!```')
with open('data.py', 'w') as data:
data.write('data = ' + str(links))
from neomodel import clear_neo4j_database, db
from neomodels import NeoFactory, ObjectFactory
from neomodels.NeoModels import LicenseModel, DatasetModel, license_filter_labels, dataset_filter_search, license_filter_sets
from neomodels.NeoModels import get_leaf_licenses, get_root_licenses, get_compliant_licenses, get_compatible_licenses
from utils.TimerDecorator import fn_timer, LOGGER
from utils.authentificator import need_auth
from utils import D3jsData
from utils import Constraints
from utils import LicenseGenerator
from utils import CSVExporter
from utils import ODRL
from utils import RDFExporter
LEVELS_FILE = "license_levels.json"
URL_VALIDATOR = URLValidator()
@require_http_methods(['GET', 'POST', 'DELETE'])
def license_path(request, graph):
if request.method == 'GET':
return get_licenses(request, graph)
elif request.method == 'POST':
return add_license(request, graph)
elif request.method == 'DELETE':
return delete_license(request)
@require_http_methods(['GET', 'POST'])
def dataset_path(request, graph):
if request.method == 'GET':
class QuickConverter(QuickConverterFile, QuickConverterURL,
QuickConverterDropboxURL, QuickConverterTextXlsForm,
QuickConverterCsvFile):
project = forms.IntegerField(required=False)
validate = URLValidator()
def clean_project(self):
project = self.cleaned_data['project']
if project is not None:
try:
self._project = Project.objects.get(pk=int(project))
except (Project.DoesNotExist, ValueError):
raise forms.ValidationError(
_(u"Unknown project id: %s" % project))
return project
def publish(self, user, id_string=None, created_by=None):
if self.is_valid():
# If a text (csv) representation of the xlsform is present,
# this will save the file and pass it instead of the 'xls_file'
# field.
if 'text_xls_form' in self.cleaned_data\
and self.cleaned_data['text_xls_form'].strip():
csv_data = self.cleaned_data['text_xls_form']
# assigning the filename to a random string (quick fix)
import random
rand_name = "uploaded_form_%s.csv" % ''.join(
random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6))
cleaned_xls_file = \
default_storage.save(
upload_to(None, rand_name, user.username),
ContentFile(csv_data))
else:
cleaned_xls_file = self.cleaned_data['xls_file']
if not cleaned_xls_file:
cleaned_url = (self.cleaned_data['xls_url'].strip()
or self.cleaned_data['dropbox_xls_url']
or self.cleaned_data['csv_url'])
cleaned_xls_file = urlparse(cleaned_url)
cleaned_xls_file = \
'_'.join(cleaned_xls_file.path.split('/')[-2:])
name, extension = os.path.splitext(cleaned_xls_file)
if extension not in VALID_FILE_EXTENSIONS:
r = requests.get(cleaned_url)
if r.headers.get('content-type') in \
VALID_XLSFORM_CONTENT_TYPES and \
r.status_code < 400:
cleaned_xls_file = get_filename(r)
cleaned_xls_file = \
upload_to(None, cleaned_xls_file, user.username)
self.validate(cleaned_url)
xls_data = ContentFile(urllib2.urlopen(cleaned_url).read())
cleaned_xls_file = \
default_storage.save(cleaned_xls_file, xls_data)
project = self.cleaned_data['project']
if project is None:
project = get_user_default_project(user)
else:
project = self._project
# publish the xls
return publish_xls_form(cleaned_xls_file, user, project, id_string,
created_by or user)
from django.core.validators import URLValidator
url_validator = URLValidator()
CHARS = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"
def encode(num, alphabet=CHARS):
if num == 0:
return alphabet[0]
arr = []
base = len(alphabet)
while num:
num, rem = divmod(num, base)
arr.append(alphabet[rem])
arr.reverse()
return ''.join(arr)
class URL(models.Model):
user_url = models.TextField(validators=[URLValidator()])
user_email = models.EmailField(blank=True)
def issue_details(request):
"""Determine the Github issue keywords of the specified Github issue or PR URL.
Todo:
* Modify the view to only use the Github API (remove BeautifulSoup).
* Simplify the view logic.
Returns:
JsonResponse: A JSON response containing the Github issue or PR keywords.
"""
from .utils import clean_bounty_url
response = {}
url = request.GET.get('url')
url_val = URLValidator()
try:
url_val(url)
except ValidationError as e:
response['message'] = 'invalid arguments'
return JsonResponse(response)
if url.lower()[:19] != 'https://github.com/':
response['message'] = 'invalid arguments'
return JsonResponse(response)
# Web format: https://github.com/jasonrhaas/slackcloud/issues/1
# API format: https://api.github.com/repos/jasonrhaas/slackcloud/issues/1
gh_api = url.replace('github.com', 'api.github.com/repos')
try:
api_response = requests.get(gh_api, auth=_AUTH)
except ValidationError:
response['message'] = 'could not pull back remote response'
return JsonResponse(response)
if api_response.status_code != 200:
response[
'message'] = f'there was a problem reaching the github api, status code {api_response.status_code}'
response['github_resopnse'] = api_response.json()
return JsonResponse(response)
try:
response = api_response.json()
body = response['body']
except (KeyError, ValueError) as e:
response['message'] = str(e)
else:
response['description'] = body.replace('\n', '').strip()
response['title'] = response['title']
keywords = []
url = request.GET.get('url')
url_val = URLValidator()
try:
url_val(url)
except ValidationError:
response['message'] = 'invalid arguments'
return JsonResponse(response)
if url.lower()[:19] != 'https://github.com/':
response['message'] = 'invalid arguments'
return JsonResponse(response)
try:
repo_url = None
url = clean_bounty_url(url)
if '/pull' in url:
repo_url = url.split('/pull')[0]
if '/issue' in url:
repo_url = url.split('/issue')[0]
split_repo_url = repo_url.split('/')
keywords.append(split_repo_url[-1])
keywords.append(split_repo_url[-2])
html_response = requests.get(repo_url, auth=_AUTH)
except (AttributeError, ValidationError):
response['message'] = 'could not pull back remote response'
return JsonResponse(response)
try:
soup = BeautifulSoup(html_response.text, 'html.parser')
eles = soup.findAll("span", {"class": "lang"})
for ele in eles:
keywords.append(ele.text)
except ValidationError:
response['message'] = 'could not parse html'
return JsonResponse(response)
try:
response['keywords'] = keywords
except Exception as e:
print(e)
response['message'] = 'could not find a title'
return JsonResponse(response)
from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from models import Url
from django.template.loader import get_template
from shortener.base62 import *
from django.template import Context
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
# Create your views here.
val = URLValidator()
def shortenUrl(request):
if request.method == "GET":
t = get_template("index.html")
return HttpResponse(t.render())
if request.method == "POST":
url = request.POST.get("urlToShorten", "")
if url == "":
return HttpResponseRedirect("/")
try:
if not ("http://" in url) or not ("https://" in url):
url = "http://" + url
val(url)
except ValidationError, e:
t = get_template("invalid.html")
return HttpResponse(t.render())
url = url.replace("http://", "")
url = url.replace("https://", "")
QS = Url.objects.all().filter(actualUrl=url)
def post(self, request, *args, **kwargs):
if not request.user.is_authenticated:
return JsonResponse({
'Status': False,
'Error': 'Log in required'
},
status=403)
if request.user.type != 'shop':
return JsonResponse(
{
'Status': False,
'Error': 'Только для магазинов'
}, status=403)
url = request.data.get('url')
if url:
validate_url = URLValidator()
try:
validate_url(url)
except ValidationError as e:
return JsonResponse({'Status': False, 'Error': str(e)})
else:
stream = get(url).content
data = load_yaml(stream, Loader=Loader)
shop, _ = Shop.objects.get_or_create(name=data['shop'],
user_id=request.user.id)
for category in data['categories']:
category_object, _ = Category.objects.get_or_create(
id=category['id'], name=category['name'])
category_object.shops.add(shop.id)
category_object.save()
ProductInfo.objects.filter(shop_id=shop.id).delete()
for item in data['goods']:
product, _ = Product.objects.get_or_create(
name=item['name'], category_id=item['category'])
product_info = ProductInfo.objects.create(
product_id=product.id,
external_id=item['id'],
model=item['model'],
price=item['price'],
price_rrc=item['price_rrc'],
quantity=item['quantity'],
shop_id=shop.id)
for name, value in item['parameters'].items():
parameter_object, _ = Parameter.objects.get_or_create(
name=name)
ProductParameter.objects.create(
product_info_id=product_info.id,
parameter_id=parameter_object.id,
value=value)
return JsonResponse({'Status': True})
return JsonResponse({
'Status': False,
'Errors': 'Не указаны все необходимые аргументы'
})
