def test_push_func():
sample_count = 10
push = {
"60": ['60' + ''.join(random.choices(string.hexdigits, k=2)) for _ in range(0, sample_count)],
"63": ['63' + ''.join(random.choices(string.hexdigits, k=8)) for _ in range(0, sample_count)],
"69": ['69' + ''.join(random.choices(string.hexdigits, k=20)) for _ in range(0, sample_count)],
"7f": ['7f' + ''.join(random.choices(string.hexdigits, k=64)) for _ in range(0, sample_count)]
}
l = len(push["7f"])
for key, value in push.items():
for v in value:
computation = comp.Computation(st.Stack())
opcode = opcodes.opcodes[key]
opcode(bc.Bytecode(v), computation.stack)
assert len(computation.stack) == opcode.produce
# test, with invalid bytecode sequence. No item to push
opcode = opcodes.opcodes["60"]
with pytest.raises(opcodes.ExceptionalHalt):
computation = comp.Computation(st.Stack())
bytecode = bc.Bytecode("60")
# we have to increment the idx of bytecode first.
# Otherwse idx is 0 and the push operation will push 60 to the stack
bytecode.__next__()
opcode(bytecode, computation.stack)
def predict(self, given=None):
if self._pred_distr is None:
self._pred_distr = dict()
keys = self.keys()
for i, k in enumerate(keys):
a = k[:-1]
b = k[-1:]
f = self._ngrams[k]
# print(":", k, a, b, f)
if a not in self._pred_distr:
self._pred_distr[a] = dict()
if b not in self._pred_distr[a]:
self._pred_distr[a][b] = f
for a in self._pred_distr.keys():
# Sum occurences
s = 0
for b in self._pred_distr[a].keys():
s += self._pred_distr[a][b]
# Normalize
for b in self._pred_distr[a].keys():
self._pred_distr[a][b] /= s
# Predict the first element in a predicted sequence
if given is None:
population = self.keys()
return random.choices(population, self.vectorize(population))[0]
else:
# print("given", given)
assert given in self._pred_distr
population = list(self._pred_distr[given].keys())
weights = [self._pred_distr[given][k] for k in population]
# assert np.isclose(np.sum(weights), 1.0)
return random.choices(population, weights)[0]
def test(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
name = pl.get_string_attrib(element, 'answers-name')
weight = pl.get_integer_attrib(element, 'weight', 1)
allow_blank = pl.get_string_attrib(element, 'allow-blank', False)
# Get correct answer
a_tru = data['correct_answers'][name]
# If correct answer is in a format generated by pl.to_json, convert it
# back to a standard type (otherwise, do nothing)
a_tru = pl.from_json(a_tru)
if allow_blank:
# no invalid answer implemented when allow-blank="true"
result = random.choices(['correct', 'incorrect'], [5, 5])[0]
else:
result = random.choices(['correct', 'incorrect', 'invalid'], [5, 5, 1])[0]
if result == 'correct':
data['raw_submitted_answers'][name] = a_tru
data['partial_scores'][name] = {'score': 1, 'weight': weight}
elif result == 'incorrect':
data['raw_submitted_answers'][name] = a_tru + str((random.randint(1, 11) * random.choice([-1, 1])))
data['partial_scores'][name] = {'score': 0, 'weight': weight}
elif result == 'invalid':
data['raw_submitted_answers'][name] = ''
data['format_errors'][name] = 'invalid'
else:
raise Exception('invalid result: %s' % result)
def create_random_name(self):
numbers = ''.join(random.choices(string.digits, k=3))
letters = ''.join(random.choices(string.ascii_uppercase, k=2))
name = letters + numbers
if name not in Robot.used_names:
Robot.used_names.append(name)
return name
return self.create_random_name()
def generate_sequence(p_unigrams, p_bigrams, num_words=100, seed_word=None):
"""Generate a random sequence of words, given unigram and bigram probabilities."""
# If seed_word is not given, pick one randomly based on unigram probabilities
if seed_word is None:
seed_word = random.choices(list(p_unigrams.keys()), weights=list(p_unigrams.values()))[0]
seq = [seed_word]
for i in range(num_words):
seq.append(random.choices(list(p_bigrams[seq[-1]].keys()), weights=list(p_bigrams[seq[-1]].values()))[0])
return seq
def test_find_it(xs):
v1, v2 = (0, 0)
if len(xs) > 1:
t = list(xs)
v1 = choices(t)[0]
t.remove(v1)
v2 = choices(t)[0]
k = v1 + v2
if len(xs) > 1:
assert find_it(xs, k)
else:
assert not find_it(xs, k)
def generateActivityMapFile(self, template = 'activity.map'):
'''Build mapfile for activity from template. Write it to a location that mapserver can see it.
The mapfile performs direct SQL queries, so we must pass all of the connection parameters for the dbAlias.
'''
filename, ext = os.path.splitext(template)
if 'mappath' in self.request.session:
logger.debug("Reusing request.session['mappath'] = %s", self.request.session['mappath'])
else:
self.request.session['mappath'] = __name__ + '_' + ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) + '.map'
logger.debug("Setting new request.session['mappath'] = %s", self.request.session['mappath'])
# mapserver_host: Hostname where 'http://<mapserver_host>/cgi-bin/mapserv?file=<url_mappath>' works
# With Apache RewriteBase rule this pattern also works for cleaner URLs:
# Allow WMS requests to any file in /dev/shm by providing a URL like /wms/ADFASDFASDF (where /dev/shm/ADFASDFASDF.map is the name of the mapfile)
# this means that the "?map=" need not be provided as part of the URL
# <Location /wms>
# Options Indexes FollowSymLinks
# RewriteEngine On
# RewriteBase '/wms/'
# RewriteRule .*/(.*) /cgi-bin/mapserv?map=/dev/shm/$1.map [QSA]
# </location>
##import pprint
##logger.debug(pprint.pformat(settings.DATABASES[self.request.META['dbAlias']]))
response = render(self.request, template, context={'mapserver_host': settings.MAPSERVER_HOST,
'list': self.itemList,
'trajectory_union_layer_string': self.trajectory_union_layer_string,
'station_union_layer_string': self.station_union_layer_string,
'wfs_title': 'WFS title for an Activity',
'map_debug_level': self.map_debug_level,
'layer_debug_level': self.layer_debug_level,
'copyright_string': 'MBARI %d' % datetime.today().year,
'dbconn': settings.MAPSERVER_DATABASES[self.request.META['dbAlias']],
'mappath': self.url_mappath,
'imagepath': settings.MAPFILE_DIR,
'STATIC_ROOT': settings.STATIC_ROOT})
try:
fh = open(self.file_mappath, 'w')
except IOError:
# In case of accessed denied error, create a new mappath and store it in the session, and open that
self.request.session['mappath'] = __name__ + '_' + ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) + '.map'
self.file_mappath = os.path.join(settings.MAPFILE_DIR, self.request.session['mappath'])
self.url_mappath = os.path.join(settings.URL_MAPFILE_DIR, self.request.session['mappath'])
fh = open(self.file_mappath, 'w')
for line in response:
##logger.info(line.decode("utf-8"))
fh.write(line.decode("utf-8"))
def select_db_for_read(weighted_dbs):
"""
Returns a randomly selected database per the weights assigned from
a list of databases. If any database is standby and its replication has
more than acceptable delay, that db is dropped from selection
Args:
weighted_dbs: a list of tuple of db and the weight.
[
("pgmain", 5),
("pgmainstandby", 5)
]
"""
# convert to a db to weight dictionary
weights_by_db = {_db: weight for _db, weight in weighted_dbs}
# filter out stale standby dbs
fresh_dbs = filter_out_stale_standbys(weights_by_db)
dbs = []
weights = []
for _db, weight in six.iteritems(weights_by_db):
if _db in fresh_dbs:
dbs.append(_db)
weights.append(weight)
if dbs:
return choices(dbs, weights=weights)[0]
def random_word(N):
return ''.join(
random.choices(
string.ascii_uppercase + string.ascii_lowercase + string.digits,
k=N
)
)
def main():
center = CallCenter()
respondents = [Respondent() for _ in range(10)]
directors = [Director() for _ in range(5)]
managers = [Manager() for _ in range(1)]
center.set_employees(Level.RESPONDENT, respondents)
center.set_employees(Level.DIRECTOR, directors)
center.set_employees(Level.MANAGER, managers)
center.initialize_queue_level_calls(Level.RESPONDENT, 20)
center.initialize_queue_level_calls(Level.DIRECTOR, 10)
center.initialize_queue_level_calls(Level.MANAGER, 3)
for employee in chain(respondents, directors, managers):
employee.start()
employee.setAvailable()
fmt = '[Status]L0:%2d L1:%2d L2:%2d\r'
while True:
if random() <= CALL_PROB:
caller = ''.join(choices(ascii_lowercase, k=5))
logging.debug('New caller: ' + caller)
center.dispatch_call(caller)
time.sleep(SLEEP_TIME)
sys.stdout.flush()
queue_calls = center.queue_calls
sys.stdout.write(fmt % (queue_calls[Level.RESPONDENT].qsize(),
queue_calls[Level.DIRECTOR].qsize(),
queue_calls[Level.MANAGER].qsize()))
def setUp(self):
"""Define the test client and other test variables."""
self.email = '{}@ungleich.ch'.format(''.join(random.choices(string.ascii_uppercase, k=10)))
self.password = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20))
self.user = CustomUser.objects.create(name='test', email=self.email,
password=self.password)
self.vm_specs = {}
self.vm_specs['cpu'] = 1
self.vm_specs['memory'] = 2
self.vm_specs['disk_size'] = 10
self.manager = OpenNebulaManager()
def test(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
name = pl.get_string_attrib(element, 'answers-name')
weight = pl.get_integer_attrib(element, 'weight', 1)
result = random.choices(['correct', 'incorrect', 'invalid'], [5, 5, 1])[0]
if result == 'correct':
data['raw_submitted_answers'][name] = str(pl.from_json(data['correct_answers'][name]))
data['partial_scores'][name] = {'score': 1, 'weight': weight}
elif result == 'incorrect':
data['raw_submitted_answers'][name] = str(pl.from_json(data['correct_answers'][name])) + ' + {:d}'.format(random.randint(1, 100))
data['partial_scores'][name] = {'score': 0, 'weight': weight}
elif result == 'invalid':
invalid_type = random.choice(['float', 'complex', 'expression', 'function', 'variable', 'syntax', 'escape', 'comment'])
if invalid_type == 'float':
data['raw_submitted_answers'][name] = 'x + 1.234'
s = 'Your answer contains the floating-point number ' + str(1.234) + '. '
s += 'All numbers must be expressed as integers (or ratios of integers). '
s += '<br><br><pre>' + phs.point_to_error('x + 1.234', 4) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'complex':
data['raw_submitted_answers'][name] = 'x + (1+2j)'
s = 'Your answer contains the complex number ' + str(2j) + '. '
s += 'All numbers must be expressed as integers (or ratios of integers). '
s += '<br><br><pre>' + phs.point_to_error('x + (1+2j)', 7) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'expression':
data['raw_submitted_answers'][name] = '1 and 0'
s = 'Your answer has an invalid expression. '
s += '<br><br><pre>' + phs.point_to_error('1 and 0', 0) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'function':
data['raw_submitted_answers'][name] = 'atan(x)'
s = 'Your answer calls an invalid function "' + 'atan' + '". '
s += '<br><br><pre>' + phs.point_to_error('atan(x)', 0) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'variable':
data['raw_submitted_answers'][name] = 'x + y'
s = 'Your answer refers to an invalid variable "' + 'y' + '". '
s += '<br><br><pre>' + phs.point_to_error('x + y', 4) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'syntax':
data['raw_submitted_answers'][name] = 'x +* 1'
s = 'Your answer has a syntax error. '
s += '<br><br><pre>' + phs.point_to_error('x +* 1', 4) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'escape':
data['raw_submitted_answers'][name] = 'x + 1\\n'
s = 'Your answer must not contain the character "\\". '
s += '<br><br><pre>' + phs.point_to_error('x + 1\\n', 5) + '</pre>'
data['format_errors'][name] = s
elif invalid_type == 'comment':
data['raw_submitted_answers'][name] = 'x # some text'
s = 'Your answer must not contain the character "#". '
s += '<br><br><pre>' + phs.point_to_error('x # some text', 2) + '</pre>'
data['format_errors'][name] = s
else:
raise Exception('invalid invalid_type: %s' % invalid_type)
else:
raise Exception('invalid result: %s' % result)
def generate_one(self):
"""Generate a single element.
Returns
-------
element
An element from the domain.
Examples
-------
>>> generator = RepellentGenerator(['a', 'b'])
>>> gen_item = generator.generate_one()
>>> gen_item in ['a', 'b']
True
"""
# Get the weights for all items in the domain
weights = [self.probability_func(self.generated[element])
for element in self.domain]
# Sample from the domain using the weights
element = random.choices(self.domain, weights=weights)[0]
# Update the generated values and return
self.generated[element] += 1
return element
def username_from_repo(self, repo):
"""Generate a username for a git repo url
e.g. minrk-binder-example-abc123
from https://github.com/minrk/binder-example.git
"""
# start with url path
print
if '://' not in repo and _ssh_repo_pat.match(repo):
# ssh url
path = repo.split(':', 1)[1]
else:
path = urlparse(repo).path
prefix = path.strip('/').replace('/', '-').lower()
if prefix.endswith('.git'):
# strip trailing .git
prefix = prefix[:-4]
if len(prefix) > 32:
# if it's long, truncate
prefix = '{}-{}'.format(prefix[:15], prefix[-15:])
# add a random suffix to avoid collisions for users on the same image
return '{}-{}'.format(prefix, ''.join(random.choices(SUFFIX_CHARS, k=SUFFIX_LENGTH)))
def test(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
name = pl.get_string_attrib(element, 'answers-name')
weight = pl.get_integer_attrib(element, 'weight', 1)
correct_key = data['correct_answers'].get(name, {'key': None}).get('key', None)
if correct_key is None:
raise Exception('could not determine correct_key')
number_answers = len(data['params'][name])
all_keys = [chr(ord('a') + i) for i in range(number_answers)]
incorrect_keys = list(set(all_keys) - set([correct_key]))
result = random.choices(['correct', 'incorrect', 'invalid'], [5, 5, 1])[0]
if result == 'correct':
data['raw_submitted_answers'][name] = data['correct_answers'][name]['key']
data['partial_scores'][name] = {'score': 1, 'weight': weight}
elif result == 'incorrect':
if len(incorrect_keys) > 0:
data['raw_submitted_answers'][name] = random.choice(incorrect_keys)
data['partial_scores'][name] = {'score': 0, 'weight': weight}
else:
# actually an invalid submission
data['raw_submitted_answers'][name] = '0'
data['format_errors'][name] = 'INVALID choice'
elif result == 'invalid':
data['raw_submitted_answers'][name] = '0'
data['format_errors'][name] = 'INVALID choice'
# FIXME: add more invalid choices
else:
raise Exception('invalid result: %s' % result)
def openpmd_file():
random_id = "".join(random.choices(string.ascii_uppercase + string.digits, k=N))
path = f"/tmp/testfile{random_id}.hdf5"
f = create_openpmd_hdf5(path)
yield path, f
f.close()
os.remove(path)
def test(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
name = pl.get_string_attrib(element, 'answers-name')
weight = pl.get_integer_attrib(element, 'weight', 1)
# Get correct answer
a_tru = data['correct_answers'][name]
# If correct answer is in a format generated by pl.to_json, convert it
# back to a standard type (otherwise, do nothing)
a_tru = pl.from_json(a_tru)
result = random.choices(['correct', 'incorrect', 'invalid'], [5, 5, 1])[0]
if result == 'correct':
data['raw_submitted_answers'][name] = str(a_tru)
data['partial_scores'][name] = {'score': 1, 'weight': weight}
elif result == 'incorrect':
data['raw_submitted_answers'][name] = str(a_tru + (random.randint(1, 11) * random.choice([-1, 1])))
data['partial_scores'][name] = {'score': 0, 'weight': weight}
elif result == 'invalid':
# FIXME: add more invalid expressions, make text of format_errors
# correct, and randomize
if random.choice([True, False]):
data['raw_submitted_answers'][name] = '1 + 2'
else:
data['raw_submitted_answers'][name] = '3.4'
data['format_errors'][name] = 'invalid'
else:
raise Exception('invalid result: %s' % result)
def setUp(self):
self.user = User.objects.create(username='******')
self.client = APIClient()
self.client.force_authenticate(user=self.user)
self.lang = Language.objects.create(
slug='en-x-demo',
name='english')
self.anthology = Anthology.objects.create(
slug='ot',
name="old testament")
self.book = Book.objects.create(
name='mark',
number=5,
slug='mrk',
anthology=self.anthology)
self.version = Version.objects.create(
slug='ulb',
name="Unlocked literal bible")
self.mode = Mode.objects.create(
slug="chk",
name="chunk",
unit=1)
self.project = Project.objects.create(
version=self.version,
mode=self.mode,
anthology=self.anthology,
language=self.lang,
book=self.book)
self.random_url = ''.join(random.choices(
string.ascii_uppercase + string.digits,
k=random.randint(1, 15)))
def command(self, bot, comm, groups):
choices = self.parse(comm['message'])
chance_of_snark = 0.05
snarks = [
"I don't know, I'm just a bot",
['Neither', 'None of them.'],
['Why not both?', 'Why not all of them?'],
[u'¿Por qué no los dos?', u'¿Por qué no los todos?'],
]
snarks += obliques
if random.random() < chance_of_snark:
# snark. ignore choices and choose something funny
snark = random.choice(snarks)
if isinstance(snarks, list):
conjugation = 0 if len(choices) == 2 else 1
choice = snark[conjugation]
else:
choice = snark
else:
# no snark, give one of the original choices
choice = random.choices(choices) + '.'
bot.reply(comm, u'{0}: {1}'.format(comm['user'], choice))
return True
def select_parent_tournament(population, scores):
""" Select a parent using the tournament. """
# choose 4 random insets
chosen = random.choices(list(enumerate(population)), k=4)
best = max(chosen, key=lambda c: scores[c[0]])
return best[1]
def random_string(length=10, lower=True, upper=False, digits=False):
pool = ''
if lower:
pool += string.ascii_lowercase
if upper:
pool += string.ascii_uppercase
if digits:
pool += string.digits
return ''.join(random.choices(pool, k=length))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
_seed = getenv("CREEPS_SEED")
if _seed is None:
_seed = ''.join(choices(ascii_letters + digits + punctuation, k=10))
self.seed = _seed
self.choices = (Bush, Creep, None)
self.weights = (15, 1, 84)
self.created_zones = {}
def create_new_mutation_id(cursor, chromosome, mutation_table):
# find the last used id and create a new one
# qry = "select icgc_mutation_id from %s " % mutation_table
# qry += "where icgc_mutation_id like 'MUT_%' order by icgc_mutation_id desc limit 1"
# ret = search_db(cursor,qry)
# ordinal = 1
# if ret:
# ordinal = int( ret[0][0].split("_")[-1].lstrip("0") ) + 1
return "MUT_%s_%s"%(chromosome,''.join(choices(string.ascii_uppercase + string.digits, k=10)))
def fix_unused_unroll(model, signal):
pred = model.predict(signal)
bc = np.bincount(pred,minlength=model.n_components)
max_id = np.argmax(bc)
max_covar_id = np.argmax(model.covars_)
ids = np.argwhere(bc == 0).flatten()
used = np.argwhere(bc != 0).flatten()
probs = bc/float(sum(bc))
mapped = {}
import random
import sklearn.mixture
ids = ids[0:len(used)]
for id in ids:
# replace_id = np.random.choice(used)
# randomly select node to clone according to its "information weight"
# replace_id = np.random.choice(model.n_components,p=probs)
replace_id = random.choices(range(model.n_components),weights=bc)[0]
mapped[id] = (replace_id, 2*bc[replace_id])
# lower prob of used node
bc[replace_id] = 0
# this will make:
# cloned states for clone fail in GMixture, and make a identical copy
# cloned states from origin to have same GMixture, and idendical copy as well
# TODO: if thats okay - store relation and avoid refitting GMixture
bc[id] = bc[replace_id]
in_trans = model.transmat_[:,id].copy()
model.transmat_[id,:] = model.transmat_[replace_id,:]
model.transmat_[replace_id,id] += model.transmat_[replace_id,replace_id]
model.transmat_[id,id] += model.transmat_[replace_id,replace_id]
model.transmat_[replace_id,replace_id] = 2e-290
# staing in giver state is forbidden
# in place of that transit to cloned state
# model.transmat_[replace_id,id] += model.transmat_[replace_id,replace_id]
# model.transmat_[replace_id,replace_id] = 0.0001
utils.normalize(model.transmat_, 1)
model.startprob_[replace_id] /= 2.
model.startprob_[id] += model.startprob_[replace_id]
model.means_[id] = model.means_[replace_id]
# diverge them slighly to cover more ground
# model.means_[replace_id] *= 1.001
model._covars_[id] = model._covars_[replace_id]
print("fixed no nodes",len(ids), mapped)
def generate(self, seed: Sequence[str]) -> Iterable[Token]:
context = list(self.tokenizer.codec.encode(seed))
context = ([self.tokenizer.codec.pad_index] * (self.context_size - len(context)) + context)[-self.context_size:]
context = array(context).reshape(1, self.context_size, 1)
while True:
predicted_distribution = self.model.predict(context).reshape(self.vocabulary_size)
sample = choices(range(self.vocabulary_size), weights=predicted_distribution)[0]
yield self.tokenizer.codec.decode_index(sample)
context = roll(context, -1, axis=1)
context[-1][-1][0] = sample
def setUp(self):
self.user = User.objects.create(username='******')
self.client = APIClient()
self.client.force_authenticate(user=self.user)
self.anthology = Anthology.objects.create(
name='New Testament',
slug='ot')
self.random_url = ''.join(random.choices(
string.ascii_uppercase + string.digits,
k=random.randint(1, 15)))
def setUp(self):
self.user = User.objects.create(username='******')
self.client = APIClient()
self.client.force_authenticate(user=self.user)
self.lang = Language.objects.create(
slug='yo',
name='yolo')
self.random_url = ''.join(random.choices(
string.ascii_uppercase + string.digits,
k=random.randint(1, 15)))
def _get_random_trump_list(trumps, weights):
for _ in range(10):
trumps_list = random.choices(trumps, weights=weights, k=get_number_trumps_per_player())
if not any(trumps_list.count(x) > 1 for x in trumps_list):
return trumps_list
logging.debug('Failed to find a stable random.choices, reverting to shuffle.')
random.shuffle(trumps)
return trumps[:get_number_trumps_per_player()]
def test(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
name = pl.get_string_attrib(element, 'answers-name')
weight = pl.get_integer_attrib(element, 'weight', 1)
allow_partial_credit = pl.get_boolean_attrib(element, 'allow-partial-credit', False)
# Get correct answer
a_tru = data['correct_answers'][name]
# If correct answer is in a format generated by pl.to_json, convert it
# back to a standard type (otherwise, do nothing)
a_tru = pl.from_json(a_tru)
# Wrap true answer in ndarray (if it already is one, this does nothing)
a_tru = np.array(a_tru)
# Throw an error if true answer is not a 2D numpy array
if a_tru.ndim != 2:
raise ValueError('true answer must be a 2D array')
else:
m, n = np.shape(a_tru)
result = random.choices(['correct', 'incorrect', 'incorrect'], [5, 5, 1])[0]
number_of_correct = 0
feedback = {}
for i in range(m):
for j in range(n):
each_entry_name = name + str(n * i + j + 1)
if result == 'correct':
data['raw_submitted_answers'][each_entry_name] = str(a_tru[i, j])
number_of_correct += 1
feedback.update({each_entry_name: 'correct'})
elif result == 'incorrect':
data['raw_submitted_answers'][each_entry_name] = str(a_tru[i, j] + (random.uniform(1, 10) * random.choice([-1, 1])))
feedback.update({each_entry_name: 'incorrect'})
elif result == 'invalid':
if random.choice([True, False]):
data['raw_submitted_answers'][each_entry_name] = '1,2'
data['format_errors'][each_entry_name] = '(Invalid format)'
else:
data['raw_submitted_answers'][name] = ''
data['format_errors'][each_entry_name] = '(Invalid blank entry)'
else:
raise Exception('invalid result: %s' % result)
if result == 'invalid':
data['format_errors'][name] = 'At least one of the entries has invalid format (empty entries or not a double precision floating point number)'
if number_of_correct == m * n:
data['partial_scores'][name] = {'score': 1, 'weight': weight}
else:
if not allow_partial_credit:
score_value = 0
else:
score_value = number_of_correct / (m * n)
data['partial_scores'][name] = {'score': score_value, 'weight': weight, 'feedback': feedback}
def genetic_drift(N, p0, G):
#G = 100 #colocar ocmo input da funcao? #numero de geracoes q vamos deixar a simulacao correr
i=0
pop_atual = []
geracoes = []
frequencias = []
while (i<2*N): # da pra melhorar isso facil #criar lista de tamanho 2N com proporcao certa de alelos
if (i <2*N*p0):
pop_atual.append(1) #alelo de frequencia p0
else:
pop_atual.append(0) #outro alelo
i+=1
#random.shuffle(pop_atual) # mistura populacao (n sei se precisa, to achando q nao)
geracoes.append( pop_atual) # faz as seguintes iteracoes para cada geracao
i=1
while (i<= G):
geracoes.append( random.choices( geracoes[i-1] , k = 2*N))
#seleciona elementos de maneira aleatoria da geracao passada ate terminar de compor a nova geracao, assim cada geracao eh salva e feita a partir da anterior
i+=1
i=1
frequencias.append( p0)
while (i < len(geracoes)):
p_i = 0
j=0
while(j < 2*N):
p_i += geracoes[i][j]
j+=1
p_i = p_i/(2*N) #numero de alelos A1 dividido pelo numero total de alelos
frequencias.append( p_i )
i+=1
return frequencias
def author(self, create, extracted, **kwargs):
authors = Author.objects.all()
for author in random.choices(authors,
k=random.randint(1, len(authors))):
self.author.add(author)
visitsQuery = select([Visit.FaceId, Visit.DateOfVisit, Visit.MoneySpent]).where(Visit.FaceId == lowestValue[0] )
results = conn.execute(visitsQuery)
print()
for row in results:
print(row)
print()
print(type(row)) # rows are of type sqlalchemy.engine.result.RowProxy
"""##3.5 In case distances are above 0.5
Create a new id and enter 128 encoding values to database
"""
import random, string
fid = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) # auto-generated id for
if(lowestValue[1] > tolerance):
print("New id added")
faceToDb = Face(
FaceId=fid,
Encoding0=imgEncode[0], Encoding1=imgEncode[1], Encoding2=imgEncode[2], Encoding3=imgEncode[3], Encoding4=imgEncode[4], Encoding5=imgEncode[5], Encoding6=imgEncode[6], Encoding7=imgEncode[7],
Encoding8=imgEncode[8], Encoding9=imgEncode[9], Encoding10=imgEncode[10], Encoding11=imgEncode[11], Encoding12=imgEncode[12], Encoding13=imgEncode[13], Encoding14=imgEncode[14], Encoding15=imgEncode[15],
Encoding16=imgEncode[16], Encoding17=imgEncode[17], Encoding18=imgEncode[18], Encoding19=imgEncode[19], Encoding20=imgEncode[20], Encoding21=imgEncode[21], Encoding22=imgEncode[22], Encoding23=imgEncode[23],
Encoding24=imgEncode[24], Encoding25=imgEncode[25], Encoding26=imgEncode[26], Encoding27=imgEncode[27], Encoding28=imgEncode[28], Encoding29=imgEncode[29], Encoding30=imgEncode[30], Encoding31=imgEncode[31],
Encoding32=imgEncode[32], Encoding33=imgEncode[33], Encoding34=imgEncode[34], Encoding35=imgEncode[35], Encoding36=imgEncode[36], Encoding37=imgEncode[37], Encoding38=imgEncode[38], Encoding39=imgEncode[39],
Encoding40=imgEncode[40], Encoding41=imgEncode[41], Encoding42=imgEncode[42], Encoding43=imgEncode[43], Encoding44=imgEncode[44], Encoding45=imgEncode[45], Encoding46=imgEncode[46], Encoding47=imgEncode[47],
Encoding48=imgEncode[48], Encoding49=imgEncode[49], Encoding50=imgEncode[50], Encoding51=imgEncode[51], Encoding52=imgEncode[52], Encoding53=imgEncode[53], Encoding54=imgEncode[54], Encoding55=imgEncode[55],
Encoding56=imgEncode[56], Encoding57=imgEncode[57], Encoding58=imgEncode[58], Encoding59=imgEncode[59], Encoding60=imgEncode[60], Encoding61=imgEncode[61], Encoding62=imgEncode[62], Encoding63=imgEncode[63],
Encoding64=imgEncode[64], Encoding65=imgEncode[65], Encoding66=imgEncode[66], Encoding67=imgEncode[67], Encoding68=imgEncode[68], Encoding69=imgEncode[69], Encoding70=imgEncode[70], Encoding71=imgEncode[71],
from flask import Flask
from flask_login import LoginManager
from flask_sqlalchemy import SQLAlchemy
import random, string
x = ''.join(random.choices(string.ascii_letters + string.digits, k=16))
app = Flask(__name__)
app.config['SECRET_KEY'] = x
app.config[
'SQLALCHEMY_DATABASE_URI'] = "postgresql://*****:*****@localhost/lab5"
app.config[
'SQLALCHEMY_TRACK_MODIFICATIONS'] = True # added just to suppress a warning
db = SQLAlchemy(app)
# Flask-Login login manager
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
app.config.from_object(__name__)
from app import views
https://github.com/sadtrxsh
================================
""", Fore.WHITE + Style.NORMAL)
print('')
time.sleep(1)
print('How Many realms you want made?')
print('')
time.sleep(1)
amount = int(input(">"))
print('')
time.sleep(1)
print('Generating')
print('')
time.sleep(2)
print('')
fix = 1
while fix <= amount:
code = ('').join(random.choices(string.ascii_letters + string.digits,
k=11))
realm_url = "https://realms.gg/"
f.write(code + '\n')
realm_code = realm_url + code
print('Code: ' + realm_code)
fix += 1
time.sleep(2)
print('')
print('Now go greifing...')
import random a = [4, 3, 7] b = [10, 20, 70] print(random.choices(a, b, k=5))
def PlayerA(weightA, n):
Player_A = random.choices(population=['Rock', 'Paper', 'Scissors'],
weights=weightA,
k=n)
return Player_A
udp_header += "ffff0000" # length | check_sum
data_end = "0000" #Type
data_end += "0000"
N = pckt_len * 2
time_exe = throughput // pckt_len
# packet_header= ip_header+udp_header
def udp_downlink(data_str):
payload_end = data_str[58:]
payload = payload_end[:-8]
print("Payload: ")
print(''.join(
[chr(int(''.join(c), 16)) for c in zip(payload[0::2], payload[1::2])]))
while 1:
res = ''.join(random.choices(string.ascii_uppercase + string.digits, k=N))
data = str(res)
# data="Siddharth"
data_str = data.encode()
data_hex = data_str.hex()
packet = ip_header + udp_header + data_hex + data_end
# test=packet.decode()
print(packet)
receive_PDCP_PDU(packet)
# udp_downlink(packet)
time.sleep(time_exe)
def rndHex(n):
return ''.join(random.choices('0123456789ABCDEF', k=n))
def get_random_lower_string() -> str:
return "".join(random.choices(string.ascii_lowercase, k=10))
def train(epoch,history):
t1 = time.time()
model.train()
losses = []
accs = []
acc= 0.0
total = 0.0
running_loss = 0.0
grapheme_root_out=0.0
vowel_diacritic_out=0.0
consonant_diacritic_out=0.0
running_acc = 0.0
running_recall = 0.0
rate = 1
if epoch<30:
rate = 1
elif epoch>=30 and rate>0.65:
rate = np.exp(-(epoch-30)/60)
else:
rate = 0.65
for idx, (inputs,labels1,labels2,labels3) in enumerate(train_loader):
inputs = inputs.to(device)
labels1 = labels1.to(device)
labels2 = labels2.to(device)
labels3 = labels3.to(device)
total += len(inputs)
choice = choices(opts, weights=[0.20, 0.30, 0.50])
# print(torch.max())
# denormalize = UnNormalize(*imagenet_stats)
# print(torch.max(denormalize(inputs)))
writer.add_images('my_image', inputs, 0)
optimizer.zero_grad()
if choice[0] == 'normal':
outputs1,outputs2,outputs3 = model(inputs.float())
loss1 = 0.7*criterion(outputs1,labels1)
loss2 = 0.20* criterion(outputs2,labels2)
loss3 = 0.10*criterion(outputs3,labels3)
loss = loss1 + loss2 + loss3
running_loss += loss.item()
elif choice[0] == 'mixup':
inputs, targets = mixup(inputs, labels1, labels2, labels3, np.random.uniform(0.8, 1.0))
outputs1, outputs2, outputs3 = model(inputs.float())
loss1, loss2, loss3 = mixup_criterion(outputs1,outputs2,outputs3, targets, rate=rate)
loss = 0.7*loss1 + 0.20*loss2 + 0.10*loss3
running_loss += loss.item()
elif choice[0] == 'cutmix':
inputs, targets = cutmix(inputs, labels1, labels2, labels3, np.random.uniform(0.8, 1.0))
outputs1, outputs2, outputs3 = model(inputs.float())
loss1, loss2, loss3 = cutmix_criterion(outputs1,outputs2,outputs3, targets, rate=rate)
loss = 0.7*loss1 + 0.20*loss2 + 0.10*loss3
running_loss += loss.item()
grapheme_root_out += (outputs1.argmax(1)==labels1).float().mean()
vowel_diacritic_out += (outputs2.argmax(1)==labels2).float().mean()
consonant_diacritic_out += (outputs3.argmax(1)==labels3).float().mean()
if apex:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
optimizer.zero_grad()
acc = running_acc/total
# scheduler.step()
elapsed = int(time.time() - t1)
eta = int(elapsed / (idx+1) * (len(train_loader)-(idx+1)))
lr = None
for param_group in optimizer.param_groups:
lr = param_group['lr']
writer.add_scalar('Learning Rate', lr, epoch*len(train_loader)+idx)
writer.add_scalar('OHEM Rate', rate, epoch)
writer.add_scalar('Loss/train', running_loss/(idx+1), epoch*len(train_loader)+idx)
writer.add_scalar('Train Accuracy/Root', grapheme_root_out/(idx+1), epoch*len(train_loader)+idx)
writer.add_scalar('Train Accuracy/Vowel', vowel_diacritic_out/(idx+1), epoch*len(train_loader)+idx)
writer.add_scalar('Train Accuracy/Consonant', consonant_diacritic_out/(idx+1), epoch*len(train_loader)+idx)
if idx%1==0:
msg = 'Epoch: {} \t Progress: {}/{} \t Loss: {:.4f} \t Time: {}s \t ETA: {}s'.format(epoch,
idx, len(train_loader), running_loss/(idx+1), elapsed, eta)
print(msg, end='\r')
# \nAcc: \t Root {:.4f} \t Vowel {:.4f} \t Consonant {:.4f}
# , grapheme_root_out/(idx+1), vowel_diacritic_out/(idx+1), consonant_diacritic_out/(idx+1)
# stdscr.addstr(0, 0, msg)
# stdscr.refresh()
losses.append(running_loss/len(train_loader))
# accs.append(running_acc/(len(train_loader)*3))
total_train_recall = running_recall/len(train_loader)
torch.cuda.empty_cache()
gc.collect()
history.loc[epoch, 'train_loss'] = losses[0]
history.loc[epoch, 'Time'] = elapsed
return total_train_recall
skills = sorted([
skills_scaled[x]
for x in rand.choice(range(len(skills_scaled)), p=skills_distr, size=3)
],
reverse=True)
# 3 unique languages
rand_lang = random.sample(languages, k=3)
rand_projects = random.sample(projects, k=3)
rand_roles = random.sample(roles, k=random.randint(1, 3))
# pad the remaining with null
rand_roles = rand_roles + ([('NULL', 'NULL')] * 3)[:3 - len(rand_roles)]
rand_password = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=30))
rand_mail = random.choice(mails)
rand_name = random.choice(names)
rand_surname = random.choice(surnames)
next_id = db.get_next_id()
# insert into main database
record = Record.Record(
rand_name,
rand_surname,
rand_mail,
rand_password,
sid=next_id,
record={
x[1]: x[0]
for x in zip([
rand_lang[0][TXT], rand_lang[1][TXT], rand_lang[2][TXT],
def random_string():
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=20))
def choice_reply(item):
reply = read_yaml("config")["event"][item]
return random.choices(list(reply.keys()), weights=list(reply.values()))[0]
def gen_verify_code(length=6):
verify_code_list = random.choices(string.digits, k=length)
return ''.join(verify_code_list)
def category(self, create, extracted, **kwargs):
categories = Category.objects.all()
for category in random.choices(categories,
k=random.randint(1, len(categories))):
self.category.add(category)
def random_choice(d):
clave_random = random.choices(list(d), weights=d.values())
clave = clave_random[0]
return clave
s1 = int(input("Enter the starting position of person 1: "))
s2 = int(input("Enter the starting position of person 2: "))
prob1 = [p1, round(float(1 - p1), 1)]
prob2 = [p2, round(float(1 - p2), 1)]
x = abs(s1 - s2)
path1 = [s1]
path2 = [s2]
check = False
pos1 = 0
pos2 = 0
time = 0 #Assumption = This time is equivalent to real time so if time = 5 then it is equal to 5 seconds.
time_histo = []
for i in range(1000):
while (check == False):
choice1 = choices(
[1, -1], prob1
) #Selects 1 & -1 with the corresponding probabilties specified in prob1 list.
choice2 = choices(
[1, -1], prob2
) #Selects 1 & -1 with the corresponding probabilties specified in prob2 list.
pos1 = path1[-1] + choice1[0]
pos2 = path2[-1] + choice2[0]
path1.append(pos1)
path2.append(pos2)
time += 1
if path1[-1] == path2[-1]:
check = True
if time == 100: #Will wait for 100seconds if path don't meet it will end
break
time_histo.append(time)
path1 = [s1]
def random_keys(self, max_keys):
"""Return a set of keys that is of size <= max_keys"""
return set(random.choices(self.keys, k=max_keys))
def __getitem__(self, idx):
# MLM
origin_txt, img_path, is_aligned, itm_prob = self.random_pair_sampling(
idx)
if self.args.img_channel == 3:
image = Image.open(os.path.join(self.data_dir,
img_path)).convert("RGB")
elif self.args.img_channel == 1:
image = Image.open(os.path.join(self.data_dir,
img_path)).convert("RGB")
image = self.transforms(image)
tokenized_sentence = self.tokenizer(
origin_txt) # ['i','ate','an','apple'], no special token
truncate_txt(tokenized_sentence, self.seq_len)
if self.args.bert_model == "albert-base-v2":
encoded_sentence = [
self.vocab_stoi[w]
if w in self.vocab_stoi else self.vocab_stoi["<unk>"]
for w in tokenized_sentence
]
else:
encoded_sentence = [
self.vocab_stoi[w]
if w in self.vocab_stoi else self.vocab_stoi["[UNK]"]
for w in tokenized_sentence
] # [178, 8756, 1126, 12075]
input_ids, txt_labels = self.random_word(encoded_sentence)
if self.disturbing_mask:
input_ids = input_ids + [self.vocab_stoi["[SEP]"]]
txt_labels_t = [-100] + txt_labels + [-100]
txt_labels_i = [-100] * (self.args.num_image_embeds + 2)
else:
input_ids = input_ids + [self.vocab_stoi["[SEP]"]]
txt_labels_t = txt_labels + [-100]
txt_labels_i = [-100] * (self.args.num_image_embeds + 2)
attn_masks_t = [1] * len(input_ids)
attn_masks_i = [1] * (self.args.num_image_embeds + 2)
if self.args.bert_model == "albert-base-v2":
padding = [
self.vocab_stoi["<pad>"]
for _ in range(self.seq_len - len(input_ids) + 1)
] # [SEP]
else:
padding = [
self.vocab_stoi["[PAD]"]
for _ in range(self.seq_len - len(input_ids) + 1)
] # [SEP]
label_padding = [
-100 for _ in range(self.seq_len - len(input_ids) + 1)
] # [SEP]
input_ids.extend(padding)
attn_masks_t.extend(padding)
txt_labels_t.extend(label_padding)
txt_labels = txt_labels_i + txt_labels_t
attn_masks = attn_masks_i + attn_masks_t # attn_masks [1, 1, 1, 1, 1, 1, 1, 1, 0, 0] -> Img_feat, Token, Pad
segment = [1 for _ in range(self.seq_len + 1)] # 2 [SEP]
cls_tok = [self.vocab_stoi["[CLS]"]]
cls_tok = torch.tensor(cls_tok)
input_ids_tensor = torch.tensor(input_ids)
txt_labels = torch.tensor(txt_labels)
segment = torch.tensor(segment)
is_aligned = torch.tensor(is_aligned)
attn_1d = torch.tensor(attn_masks)
full_attn = torch.tensor(
(attn_masks_i + attn_masks_t),
dtype=torch.long).unsqueeze(0).expand(self.total_len,
self.total_len).clone()
extended_attn_masks = torch.zeros(self.total_len,
self.total_len,
dtype=torch.long)
second_st, second_end = self.args.num_image_embeds + 2, self.args.num_image_embeds + 2 + len(
input_ids)
extended_attn_masks[:, :self.args.num_image_embeds + 2].fill_(1)
extended_attn_masks[second_st:second_end, second_st:second_end].copy_(
self._tril_matrix[:second_end - second_st, :second_end -
second_st])
s2s_attn = extended_attn_masks
mixed_lst = [full_attn, s2s_attn]
if self.args.Mixed:
# print('Mixed attn mask')
assert (self.args.s2s_prob + self.args.bi_prob) == 1.0
attn_masks_tensor = random.choices(
mixed_lst, weights=[self.args.bi_prob, self.args.s2s_prob])[0]
# print(f'S2S {self.args.s2s_prob} vs Bi {self.args.bi_prob}')
elif self.args.BAR_attn:
# print('BAR_attn attn mask')
extended_attn_masks[:self.args.num_image_embeds + 2, :].fill_(1)
attn_masks_tensor = extended_attn_masks
elif self.args.disturbing_mask:
baseline_attn = torch.zeros(self.total_len,
self.total_len,
dtype=torch.long)
baseline_attn[:self.args.num_image_embeds +
2, :self.args.num_image_embeds + 2].fill_(1)
baseline_attn[self.args.num_image_embeds + 2:,
self.args.num_image_embeds + 2:].fill_(1)
attn_masks_tensor = baseline_attn
else:
if self.args.attn_1d:
# print('1d_bidirectional attn mask')
attn_masks_tensor = attn_1d # '1d attention mask'
else:
# print('full_bidirecitonal attn mask')
attn_masks_tensor = full_attn # 'Full attention mask'
sep_tok = [self.vocab_stoi["[SEP]"]]
sep_tok = torch.tensor(sep_tok)
return cls_tok, input_ids_tensor, txt_labels, attn_masks_tensor, image, segment, is_aligned, sep_tok, itm_prob
def roulette_selection(candidates):
return random.choices(candidates,
[min(0.0, x.fitness) for x in candidates])
def tele_Number(cls):
import random
cls.tele_Num = random.choices(__class__.l, k=1)
cls.tele_BodyNum = random.choices("0123456789", k=8)
print("".join(cls.tele_Num + cls.tele_BodyNum))
def crawler():
# Establishing connection with mongodb
client = pymongo.MongoClient(config["localhost"], config["port_num"])
db = client[config['database_name']]
col = db[config['collection_name']]
# starting scraping
if col.count_documents(
{}) == 0: # if collection is empty : scrape flinkhub.com
links_list1 = []
headers1 = {
'User-Agent': config['user_agent'],
}
try: # send request
logger.debug("Making HTTP GET request: " + config['host_name'])
r1 = requests.get(config['host_name'], headers=headers1)
res1 = r1.text
logger.debug("Got HTML source, content length = " + str(len(res1)))
except: # if cannot request url
logger.exception("Failed to get HTML source from " +
config['host_name'])
traceback.print_exc()
return links_list1
logger.debug("Extracting links from the HTML")
soup1 = BeautifulSoup(
res1, 'html.parser') # converting request to soup object
# saving html content to a .txt file
try:
file_name1 = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=16))
file_name1 = file_name1 + '.txt'
file_path1 = os.path.join(os.getcwd(), config["file_dir"],
file_name1)
text_file1 = open(file_path1, "w")
n1 = text_file1.write(str(soup1))
text_file1.close()
except:
logger.exception("Cannot write link in a file.")
if 'Content-Length' in r1.headers:
new_doc = {
"link": config["host_name"],
"source_link": None,
"is_crawled": True,
"last_crawl_date": datetime.datetime.utcnow(),
"response_status": r1.status_code,
"content_type": r1.headers['Content-Type'],
"con_length": r1.headers['Content-Length'],
"file_path": file_path1,
"created_at": datetime.datetime.utcnow(),
}
else:
new_doc = {
"link": config["host_name"],
"source_link": None,
"is_crawled": True,
"last_crawl_date": datetime.datetime.utcnow(),
"response_status": r1.status_code,
"content_type": r1.headers['Content-Type'],
"con_length": len(r1.content),
"file_path": file_path1,
"created_at": datetime.datetime.utcnow(),
}
col.insert_one(new_doc) # inserting original link to a document
links1 = soup1.find_all("a") # finding all a tags
for link1 in links1: # iterating over all the links
temp1 = link1.get('href') # getting url
if temp1 not in links_list1: # if link was not scraped in the same cycle
links_list1.append(temp1)
# checking validity of link
temp_parse1 = urllib.parse.urlparse(temp1)
netloc_bool1 = bool(temp_parse1.netloc)
scheme_bool1 = bool(temp_parse1.scheme)
if netloc_bool1:
if scheme_bool1:
# if link is valid and absolute url
actual_link1 = temp1
query1 = {"link": actual_link1}
myq1 = col.find(query1)
if myq1.count == 0:
temp_doc1 = {
"link": actual_link1,
"source_link": config['host_name'],
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc1) # adding link to a document
else:
print(temp1 + " already exists in database."
) # if link already exists in database
else:
print(temp1 +
" link not valid") # if link is not valid
else:
# if link is a relative url
actual_link2 = urllib.parse.urljoin(
config['host_name'], temp1)
netloc_bool2 = bool(urllib.parse.urlparse(actual_link2))
scheme_bool2 = bool(urllib.parse.urlparse(actual_link2))
if netloc_bool2 and scheme_bool2: # if relative url is valid
query2 = {"link": actual_link2}
if col.count_documents(
query2
) == 0: # if link doesn't exist in collection already
temp_doc1 = {
"link": actual_link2,
"source_link": config['host_name'],
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc1) # inserting link to collection
else:
print(str(actual_link2) + " already exists."
) # if link already exists in collection
else:
print(actual_link2 +
" not valid") # if link is not valid
else:
print(temp1 + " Link already scraped"
) # if link is already scraped in the same cycle
return links1
else: # if there exist some links in the collection already
if col.count_documents({
"is_crawled": False
}) > 0: # if there exist some documents which are not crawled
# picking a random link from the collection to be scraped
num1 = col.count_documents({"is_crawled": False})
random1 = math.floor(random.random() * num1)
cursor_doc = col.find({"is_crawled": False}).limit(1).skip(random1)
for curs in cursor_doc:
doc = curs
links_list2 = []
og_link = doc['link']
headers2 = {
'User-Agent': config['user_agent'],
}
try: # requesting link
logger.debug("Making HTTP GET request: " + og_link)
r2 = requests.get(og_link, headers=headers2)
res2 = r2.text
logger.debug("Got HTML source, content length = " +
str(len(res2)))
except:
logger.exception("Failed to get HTML source from " + og_link)
traceback.print_exc()
return links_list2
logger.debug("Extracting links from the HTML")
soup2 = BeautifulSoup(
res2, 'html.parser') # converting request to a soup object
# saving html content to a file
try:
file_name2 = ''.join(
random.choices(string.ascii_uppercase + string.digits,
k=16))
file_name2 = file_name2 + '.txt'
file_path_2 = os.path.join(os.getcwd(), config['file_dir'],
file_name2)
text_file2 = open(file_path_2, "w")
n2 = text_file2.write(str(soup2))
text_file2.close()
except:
logger.exception("Cannot write link in a file.")
if 'Content-Length' in r2.headers:
updated_doc = {
"is_crawled": True,
"last_crawl_date": datetime.datetime.utcnow(),
"response_status": r2.status_code,
"content_type": r2.headers['Content-Type'],
"con_length": r2.headers['Content-Length'],
"file_path": file_path_2,
}
else:
updated_doc = {
"is_crawled": True,
"last_crawl_date": datetime.datetime.utcnow(),
"response_status": r2.status_code,
"content_type": r2.headers['Content-Type'],
"con_length": len(r2.content),
"file_path": file_path_2,
}
col.update_one(
doc,
{"$set": updated_doc}) # updating link which was just scraped
links2 = soup2.find_all("a") # converting request to a soup object
for link2 in links2:
temp2 = link2.get('href') # getting link from a tag
if temp2 not in links_list2: # itertaing through links
links_list2.append(temp2)
# checking validity of links
temp_parse3 = urllib.parse.urlparse(temp2)
netloc_bool3 = bool(temp_parse3.netloc)
scheme_bool3 = bool(temp_parse3.scheme)
if netloc_bool3:
if scheme_bool3:
# valid absolute link
actual_link3 = temp2
query3 = {"link": actual_link3}
if col.count_documents(query3) == 0:
temp_doc = {
"link": actual_link3,
"source_link": og_link,
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc) # adding link to the collection
else:
print(temp2 + " already exists."
) # if link already exists in collection
else: # if link is not valid
print(temp2 + " link not valid")
else:
# link is a relative link
actual_link4 = urllib.parse.urljoin(og_link, temp2)
netloc_bool4 = bool(
urllib.parse.urlparse(actual_link4))
scheme_bool4 = bool(
urllib.parse.urlparse(actual_link4))
if netloc_bool4 and scheme_bool4:
# valid relative link
query4 = {"link": actual_link4}
if col.count_documents(
query4
) == 0: # checking for existence of link in collection
temp_doc = {
"link": actual_link4,
"source_link": og_link,
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc) # adding link to the collection
else:
print(actual_link4 + " already exists."
) # link already exists in collection
else:
print(actual_link4 +
" not valid") # link is not valid
return links2 # return list of links found
else: # if there are no links which are not crawled yet
valid_docs = col.find({})
# finding links which were not crawled in last 24 hours
time_dif = datetime.timedelta(days=1)
greater_than_24_docs = []
for single_doc in valid_docs:
if single_doc["last_crawl_date"] > time_dif:
greater_than_24_docs.append(single_doc)
num2 = len(greater_than_24_docs)
# picking a random link out of those links which were not crawled in last 24 hours
random2 = random.randint(0, num2 - 1)
doc = greater_than_24_docs[random2]
links_list2 = []
og_link = doc.link
headers2 = {
'User-Agent': config['user_agent'],
}
# making a https request
try:
logger.debug("Making HTTP GET request: " + og_link)
r2 = requests.get(og_link, headers=headers2)
res2 = r2.text
logger.debug("Got HTML source, content length = " +
str(len(res2)))
except:
logger.exception("Failed to get HTML source from " + og_link)
traceback.print_exc()
return links_list2
logger.debug("Extracting links from the HTML")
soup2 = BeautifulSoup(
res2, 'html.parser') # turning request into soup object
try:
# writing html content to a txt file
file_name2 = ''.join(
random.choices(string.ascii_uppercase + string.digits,
k=16))
file_name2 = file_name2 + '.txt'
file_path2 = os.path.join(os.getcwd(), config['file_dir'],
file_name2)
text_file2 = open(file_path2, "w")
n2 = text_file2.write(str(soup2))
text_file2.close()
except:
logger.exception("Cannot write link in a file.")
if 'Content-Length' in r2.headers:
updated_doc = {
"is_crawled": True,
"last_crawl_date": datetime.date.today(),
"response_status": r2.status_code,
"content_type": r2.headers['Content-Type'],
"con_length": r2.headers['Content-Length'],
"file_path": file_path2,
}
else:
updated_doc = {
"is_crawled": True,
"last_crawl_date": datetime.date.today(),
"response_status": r2.status_code,
"content_type": r2.headers['Content-Type'],
"con_length": len(r2.content),
"file_path": file_path2,
}
col.update_one(doc,
{"$set": updated_doc
}) # updating the recently crawled link document
links2 = soup2.find_all("a") # finding all anchor tags
for link2 in links2: # iterating through a tags
temp2 = link2.get('href') # geting the link from a tag
if temp2 not in links_list2: # if link wasn't found in this cycle
links_list2.append(temp2)
# checking for validity of link
temp_parse3 = urllib.parse.urlparse(temp2)
netloc_bool3 = bool(temp_parse3.netloc)
scheme_bool3 = bool(temp_parse3.scheme)
if netloc_bool3:
if scheme_bool3:
# link is absolute url and valid
actual_link3 = temp2
query3 = {"link": actual_link3}
if col.count_documents(query3) == 0:
temp_doc = {
"link": actual_link3,
"source_link": og_link,
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc) # adding link to the collection
else:
print(
temp2 + " already exists."
) # if link already exists in the collection
else:
print(temp2 +
" link not valid") # link is not valid
else:
# link is a relative link
actual_link4 = urllib.parse.urljoin(og_link, temp2)
netloc_bool4 = bool(
urllib.parse.urlparse(actual_link4))
scheme_bool4 = bool(
urllib.parse.urlparse(actual_link4))
if netloc_bool4 and scheme_bool4:
# link is relative and valid
query4 = {"link": actual_link4}
if col.count_documents(query4) == 0:
temp_doc = {
"link": actual_link4,
"source_link": og_link,
"is_crawled": False,
"last_crawl_date": None,
"response_status": None,
"content_type": None,
"con_length": None,
"file_path": None,
"created_at": datetime.datetime.utcnow()
}
col.insert_one(
temp_doc
) # adding link document to collection
else:
print(actual_link4 + " already exists."
) # link already exists in collection
else:
print(actual_link4 +
" not valid") # link is not valid
return link2
def generate_random_str(length=16):
return ''.join(
random.choices(string.ascii_letters + string.digits, k=length))
random.seed(a=None, version=2) random.getstate() random.setstate(state) # byte random.randbytes(n) # int random.randrange(stop) random.randrange(start, stop, ?step) random.randint(a, b) random.getrandbits(k # sequence random.choice(seq) random.choices(population, weights=None, *, cum_weights=None, k=1) random.shuffle(x, ?random) random.sample(population, k, *, counts=None) # real-valued distributions random.random() random.uniform(a, b) random.triangular(low, high, mode) random.betavariate(alpha, beta) random.expovariate(lambd) random.gammavariate(alpha, beta) random.gauss(mu, sigma) random.lognormvariate(mu, sigma) random.normalvariate(mu, sigma) random.vonmisesvariate(mu, kappa) random.paretovariate(alpha)
import random
# выборка с заменой
rand = [2, 12, 7, 4, 3, 11, 15]
test_ticket = []
# Выбираем случайное количество значений из списка
# Без повторений random.sample
# Можно random.choices тогда повторяются
win_ticket = random.choices(rand, k=2)
print("Эта комбинация выиграла", win_ticket)
rand1 = [2, 12, 7, 4, 3, 11, 15]
while rand1:
test = random.choices(rand1, k=2)
print(test)
if test == win_ticket:
print('Вот она', test)
break
def edit_keyname(path_keyfile):
while os.path.isfile(path_keyfile):
letters = string.ascii_letters
rnd = "".join(random.choices(letters, k=4))
path_keyfile = str(path_keyfile + rnd)
return path_keyfile
total_loss = 0
seen_instances = 0
losses = []
# train network
for epoch in range(epochs):
if epoch % 7 == 0 and epoch != 0:
trainer.learning_rate = trainer.learning_rate / 2
print("starting epoch: ", epoch)
dy.renew_cg()
total_loss = 0
seen_instances = 0
for i in range(rounds_per_epoch):
dy.renew_cg()
# sampling the arrays at the same indexes
losses = []
for inp, lbl in random.choices(train_encoded, k=batch_size):
loss = network.create_network_return_loss(inp, lbl)
losses.append(loss)
batch_loss = dy.esum(losses) / batch_size
if i % 4095 == 0 and i != 0:
print(
'loss:',
batch_loss.npvalue()[0]) # this calls forward on the batch
batch_loss.backward()
trainer.update()
if True:
# checking accuracy on dev set
y_predicted = [
network.create_network_return_best(x[0]) for x in test_encoded
]
def getAnimals(self, count):
categ = self.category.createCategory()
yourchoices = random.choices(categ, k=count)
for key in yourchoices:
self.category.displayData(key)
def set_jurisdiction(self, city):
""" set the jurisdiction based on where the city is """
state = cities[city]['state']
if state == 'Northern Ireland':
jurisdiction = random.choice(['Northern Ireland', 'Both'])
elif state == 'Republic of Ireland':
jurisdiction = random.choice(['Republic of Ireland', 'Both'])
return jurisdiction
if __name__ == '__main__':
# make a list to store each users's response
allResponses = []
for i in range(50):
# generate a random org name
this_orgName = ''.join(random.choices(string.ascii_uppercase + string.digits, k=3))
thisResponse = Response(orgName=this_orgName)
allResponses.append(thisResponse.__dict__)
# write the responses to a json file
with open('fakeData_large.json', 'w') as outputFile:
json.dump(allResponses, outputFile, indent=3)
def address_default():
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=9))
