def test_is_blocked(self):
source = Source(id='x')
self.assertFalse(source.is_blocked({'author': {'numeric_id': '1'}}))
source = Source(id='x', blocked_ids = ['1', '2'])
self.assertTrue(source.is_blocked({'author': {'numeric_id': '1'}}))
self.assertFalse(source.is_blocked({'object': {'actor': {'numeric_id': '3'}}}))
def create():
filesystem_id = current_app.crypto_util.hash_codename(
session['codename'])
source = Source(filesystem_id, current_app.crypto_util.display_id())
db.session.add(source)
try:
db.session.commit()
except IntegrityError as e:
db.session.rollback()
current_app.logger.error(
"Attempt to create a source with duplicate codename: %s" %
(e, ))
# Issue 2386: don't log in on duplicates
del session['codename']
# Issue 4361: Delete 'logged_in' if it's in the session
try:
del session['logged_in']
except KeyError:
pass
abort(500)
else:
os.mkdir(current_app.storage.path(filesystem_id))
session['logged_in'] = True
return redirect(url_for('.lookup'))
def setup(self):
"""
Set up method that will run before every Test
"""
self.new_source = Source("nana", "igihe",
"Test to see if sources is being known",
"TestUrl", "sport")
def test_source_insert_3(self):
s = Source(id_num='102', id_name='test-src3', language='English', \
description='Guaranteed best source for accuracy and truthiness', smallLogoURL='http://assets3.onionstatic.com/onionstatic/onion/static/images/onion_fb_placeholder.png', mediumLogoURL='the_onion.gif', \
largeLogoURL='the_onion2.img', category='The Truth', external_link='http://www.theonion.com/',name='The Onion',\
region='United States', country='United States')
db.session.add(s)
db.session.commit()
r = db.session.query(Source).filter(Source.id_num == '102').first()
self.assertEqual(r.id_num, '102')
self.assertEqual(r.id_name, 'test-src3')
self.assertEqual(r.language, 'English')
self.assertEqual(r.description,
'Guaranteed best source for accuracy and truthiness')
self.assertEqual(
r.smallLogoURL,
'http://assets3.onionstatic.com/onionstatic/onion/static/images/onion_fb_placeholder.png'
)
self.assertEqual(r.mediumLogoURL, 'the_onion.gif')
self.assertEqual(r.largeLogoURL, 'the_onion2.img')
self.assertEqual(r.category, 'The Truth')
self.assertEqual(r.external_link, 'http://www.theonion.com/')
self.assertEqual(r.name, 'The Onion')
self.assertEqual(r.region, 'United States')
self.assertEqual(r.country, 'United States')
db.session.query(Source).filter(Source.id_num == '102').delete()
db.session.commit()
def ready(self):
"""
Находит все источники брокера и
в соответствии с полученным списком обновляет таблицу бд
"""
from models import Source
# Получаем данные по всем источникам в модуле
module_sources = get_data_sources()
if not module_sources:
raise ModuleSourcesExceptions('Sources in module were not found')
module_sources_names = set(module_sources.keys())
try:
# Выбираем источники, которые уже есть в бд
db_sources_names = set(Source.objects.all().values_list('source',
flat=True))
# Удаляем источники, которых нет в модуле
Source.objects.filter(source__in=(db_sources_names -
module_sources_names)).delete()
# Добавляем источники, которых нет в бд
Source.objects.bulk_create([
Source(source=source_name,
type_source=module_sources[source_name]['type'])
for source_name in (module_sources_names - db_sources_names)
])
except ProgrammingError:
warnings.warn('Only if deployed')
def test_source_insert_2(self):
s = Source(id_num='101', id_name='test-src2', language='latin', \
description='official news source of the roman empire', smallLogoURL='small_logo.jpg', mediumLogoURL='medium_logo.jpg', \
largeLogoURL='large_logo.jpg', category='history', external_link='barbarians-are-invading.com',name='Roma News',\
region='Mediterranean Sea', country='Italia')
db.session.add(s)
db.session.commit()
r = db.session.query(Source).filter(
Source.external_link == 'barbarians-are-invading.com').first()
self.assertEqual(r.id_num, '101')
self.assertEqual(r.id_name, 'test-src2')
self.assertEqual(r.language, 'latin')
self.assertEqual(r.description,
'official news source of the roman empire')
self.assertEqual(r.smallLogoURL, 'small_logo.jpg')
self.assertEqual(r.mediumLogoURL, 'medium_logo.jpg')
self.assertEqual(r.largeLogoURL, 'large_logo.jpg')
self.assertEqual(r.category, 'history')
self.assertEqual(r.external_link, 'barbarians-are-invading.com')
self.assertEqual(r.name, 'Roma News')
self.assertEqual(r.region, 'Mediterranean Sea')
self.assertEqual(r.country, 'Italia')
db.session.query(Source).filter(
Source.external_link == 'barbarians-are-invading.com').delete()
db.session.commit()
def test_source_insert_1(self):
s = Source(id_num='100', id_name='test-src1', language='elvish', \
description='official news source of middle-earth', smallLogoURL='small.png', mediumLogoURL='medium.png', \
largeLogoURL='large.png', category='general', external_link='some-link.com',name='Middle-Earth News',\
region='fantasy', country='Mordor')
db.session.add(s)
db.session.commit()
r = db.session.query(Source).filter(
Source.name == 'Middle-Earth News').first()
self.assertEqual(r.id_num, '100')
self.assertEqual(r.id_name, 'test-src1')
self.assertEqual(r.language, 'elvish')
self.assertEqual(r.description, 'official news source of middle-earth')
self.assertEqual(r.smallLogoURL, 'small.png')
self.assertEqual(r.mediumLogoURL, 'medium.png')
self.assertEqual(r.largeLogoURL, 'large.png')
self.assertEqual(r.category, 'general')
self.assertEqual(r.external_link, 'some-link.com')
self.assertEqual(r.name, 'Middle-Earth News')
self.assertEqual(r.region, 'fantasy')
self.assertEqual(r.country, 'Mordor')
db.session.query(Source).filter(
Source.name == 'Middle-Earth News').delete()
db.session.commit()
def source_create(request, template_name='source/source_form.html'):
if request.method == 'POST':
form = SkipperForm(request.POST)
if form.is_valid():
source = Source()
source.text = form.cleaned_data['text']
source.title = form.cleaned_data['title']
source.owner = request.user.first_name + ' ' + request.user.last_name
source.save()
# Now build it.
build(user=request.user, sourceobj=source)
cache.clear()
return redirect('source_list')
else:
return render(request, template_name, {'form': form})
else:
default = '''
coordinator "Name" "Email" "Url"
contact "Email"
description {
}
'''
form = SkipperForm({'text': default, 'title': 'New Role'})
return render(request, template_name, {'form': form})
def setUp(self):
'''
Inbuilt function that runs before each test is executed
'''
self.new_source = Source(
"abc-news", "ABC News",
"Your trusted source for breaking news, analysis, exclusive interviews, headlines, and videos at ABCNews.com.",
"https://abcnews.go.com", "general", "en", "us")
def create_source_and_submissions(
source_index,
source_count,
num_submissions=2,
num_replies=2,
journalist_who_replied=None # noqa: W605, E501
):
# Store source in database
codename = current_app.crypto_util.genrandomid()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_designation = current_app.crypto_util.display_id()
source = Source(filesystem_id, journalist_designation)
source.pending = False
db.session.add(source)
db.session.commit()
# Generate submissions directory and generate source key
os.mkdir(current_app.storage.path(source.filesystem_id))
current_app.crypto_util.genkeypair(source.filesystem_id, codename)
# Generate some test submissions
for _ in range(num_submissions):
source.interaction_count += 1
submission_text = next(submissions)
fpath = current_app.storage.save_message_submission(
source.filesystem_id, source.interaction_count,
source.journalist_filename, submission_text)
source.last_updated = datetime.datetime.utcnow()
submission = Submission(source, fpath)
db.session.add(submission)
# Generate some test replies
for _ in range(num_replies):
source.interaction_count += 1
fname = "{}-{}-reply.gpg".format(source.interaction_count,
source.journalist_filename)
current_app.crypto_util.encrypt(
next(replies), [
current_app.crypto_util.get_fingerprint(source.filesystem_id),
config.JOURNALIST_KEY
], current_app.storage.path(source.filesystem_id, fname))
if not journalist_who_replied:
journalist = Journalist.query.first()
else:
journalist = journalist_who_replied
reply = Reply(journalist, source, fname)
db.session.add(reply)
db.session.flush()
seen_reply = SeenReply(reply_id=reply.id, journalist_id=journalist.id)
db.session.add(seen_reply)
db.session.commit()
print("Test source {}/{} (codename: '{}', journalist designation '{}') "
"added with {} submissions and {} replies".format(
source_index, source_count, codename, journalist_designation,
num_submissions, num_replies))
def test_is_beta_user(self):
source = Source(id='x')
self.assertFalse(source.is_beta_user())
self.mox.stubs.Set(util, 'BETA_USER_PATHS', set())
self.assertFalse(source.is_beta_user())
self.mox.stubs.Set(util, 'BETA_USER_PATHS', set([source.bridgy_path()]))
self.assertTrue(source.is_beta_user())
def getOrAddSource(name, session):
srcs = session.query(Source).filter(Source.name == name).all()
if (len(srcs) != 0):
assert len(srcs) == 1
return srcs[0]
s = Source(name=name)
session.add(s)
session.commit()
return s
def setUp(self):
"""
set up method that runs before test
"""
self.new_source = Source("", "", "", "")
def test_instance(self) :
'''
Test if the self.new_source object is an instance of the Source class
'''
self.assertTrue(isinstance(self.new_source,Source))
def get_source_from_rss(rss_url):
"""
Given an rss url, returns a dictionary with rss_url, homepage_url and name. Note: If these are not defined in RSS,
it is possible to raise an AttributeError.
:param rss_url: the url for the rss to parse
:return: A database Source object with information from the rss.
"""
feed = feedparser.parse(rss_url)
homepage_url = feed['feed']['link']
name = feed['feed']['title']
return Source(rss_url=rss_url, homepage_url=homepage_url, name=name)
def callback():
print(request.is_json)
content = request.get_json()
print content['src'], content['dst'], content['text']
src = content['src']
dst = content['dst']
text = content['text']
src_data = Source(src, dst, text)
db.session.add(src_data)
db.session.commit()
return ' The JSON posted successfully '
def new_source(self) -> None:
codename = current_app.crypto_util.genrandomid()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_designation = current_app.crypto_util.display_id()
source = Source(filesystem_id, journalist_designation)
db.session.add(source)
db.session.flush()
# Generate submissions directory and generate source key
os.mkdir(current_app.storage.path(source.filesystem_id))
current_app.crypto_util.genkeypair(source.filesystem_id, codename)
self.sources.append(source.id)
def _get_or_create_lamoda_source():
source = Source.query.filter_by(name='lamoda.ru').first()
if source is None:
source = Source(name='lamoda.ru',
site_url='http://www.lamoda.ru/',
logo_url='http://www.lamoda.ru/social-logo.jpg',
is_active=True)
db.session.add(source)
db.session.commit()
return source
def create() -> werkzeug.Response:
if session.get('logged_in', False):
flash(
gettext(
"You are already logged in. Please verify your codename above as it "
+
"may differ from the one displayed on the previous page."),
'notification')
else:
tab_id = request.form['tab_id']
codename = session['codenames'][tab_id]
session['codename'] = codename
del session['codenames']
filesystem_id = current_app.crypto_util.hash_codename(codename)
try:
source = Source(filesystem_id,
current_app.crypto_util.display_id())
except ValueError as e:
current_app.logger.error(e)
flash(
gettext(
"There was a temporary problem creating your account. "
"Please try again."), 'error')
return redirect(url_for('.index'))
db.session.add(source)
try:
db.session.commit()
except IntegrityError as e:
db.session.rollback()
current_app.logger.error(
"Attempt to create a source with duplicate codename: %s" %
(e, ))
# Issue 2386: don't log in on duplicates
del session['codename']
# Issue 4361: Delete 'logged_in' if it's in the session
try:
del session['logged_in']
except KeyError:
pass
abort(500)
else:
os.mkdir(current_app.storage.path(filesystem_id))
session['logged_in'] = True
return redirect(url_for('.lookup'))
def new_source(self):
fid_len = random.randint(4, 32)
designation_len = random.randint(4, 32)
source = Source(
random_chars(fid_len, nullable=False,
chars=string.ascii_lowercase),
random_chars(designation_len, nullable=False))
source.flagged = bool_or_none()
source.last_updated = random_datetime(nullable=False)
source.pending = False
db.session.add(source)
db.session.flush()
self.sources.append(source.id)
def enterRole(self, ctx):
title = self.__strToken(ctx.QUOTE())
inputstream = ctx.start.getInputStream()
start = ctx.getChild(3).start.start
stop = ctx.getChild(3).stop.stop
text = inputstream.getText(start, stop)
src = Source(
title=title,
text=text,
owner=self.user.first_name + ' ' + self.user.last_name,
)
src.save()
self.context.append(src)
def source_update(request, pk, template_name='source/source_form.html'):
next_page = request.GET.get('next', None)
source = Source.objects.get(title=pk)
if source == None:
raise Http404("Source does not exist")
if request.method == 'POST':
form = SkipperForm(request.POST)
if form.is_valid():
try:
with transaction.atomic():
# There is no update, sources must be deleted and remade.
# on_delete=CASCADE is emulated, but it works when my objects don't suck.
source.delete()
# Create a new one
source = Source()
source.title = pk
source.text = form.cleaned_data['text']
source.owner = request.user.first_name + ' ' + request.user.last_name
source.save()
# Now build it.
build(user=request.user, sourceobj=source)
except IntegrityError as e:
print "Transaction error:", e
cache.clear()
if form.cleaned_data['next'] != "":
return redirect('jobs', form.cleaned_data['next'])
else:
return redirect('source_list')
else:
return render(request, template_name, {
'title': pk,
'form': form,
'next': next_page
})
else:
form = SkipperForm({'text': source.text, 'title': source.title})
return render(request, template_name, {
'title': pk,
'form': form,
'next': next_page
})
def add_source():
if request.is_json:
try:
content = request.get_json()
to_add = Source(content['short_hand'], content['rss'])
try:
db.session.add(to_add)
db.session.commit()
return flask.Response(status=201)
except:
return flask.Response(status=409)
except:
return flask.Response(status=401)
else:
return flask.Response(status=400)
def upload():
uf = request.files['input-b1']
savepath, new_filename = uploadfile(uf, 'index')
source = Source(path=savepath,filename=new_filename)
user_id = session.get('user_id')
user = User.query.filter(User.id == user_id).first()
source.owner = user
try:
db.session.add(source)
db.session.commit()
except Exception as reason:
s=str(reason)
list = re.split(r'[()]+', s)
flash(u'Upload Failed：(%s)(%%s)'%list[1]%list[3])
return redirect(url_for('index'))
else:
flash(u'Upload Succ')
return redirect(url_for('index'))
def parse_sources():
with open(
"/var/www/html/cs373-idb/data/api_data/final_data/final_sources.json"
) as f:
src_json = json.load(f)
count = 0
for src in src_json:
s = Source(id_num=src['id_num'], id_name=src['id_name'], language=src['language'], \
description=src['description'], smallLogoURL=src['smallLogoURL'], mediumLogoURL=src['mediumLogoURL'], \
largeLogoURL=src['largeLogoURL'], category=src['category'], external_link=src['external_link'],name=src['name'],\
region=src['region'], country=src['country'])
db.session.add(s)
count += 1
#print("Source " + s.id_num + " added to db")
db.session.commit()
return count
def create_source_and_submissions(num_submissions=2, num_replies=2):
# Store source in database
codename = current_app.crypto_util.genrandomid()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_designation = current_app.crypto_util.display_id()
source = Source(filesystem_id, journalist_designation)
source.pending = False
db.session.add(source)
db.session.commit()
# Generate submissions directory and generate source key
os.mkdir(current_app.storage.path(source.filesystem_id))
current_app.crypto_util.genkeypair(source.filesystem_id, codename)
# Generate some test submissions
for _ in range(num_submissions):
source.interaction_count += 1
fpath = current_app.storage.save_message_submission(
source.filesystem_id, source.interaction_count,
source.journalist_filename, 'test submission!')
source.last_updated = datetime.datetime.utcnow()
submission = Submission(source, fpath)
db.session.add(submission)
# Generate some test replies
for _ in range(num_replies):
source.interaction_count += 1
fname = "{}-{}-reply.gpg".format(source.interaction_count,
source.journalist_filename)
current_app.crypto_util.encrypt(
'this is a test reply!', [
current_app.crypto_util.getkey(source.filesystem_id),
config.JOURNALIST_KEY
], current_app.storage.path(source.filesystem_id, fname))
journalist = Journalist.query.first()
reply = Reply(journalist, source, fname)
db.session.add(reply)
db.session.commit()
print("Test source (codename: '{}', journalist designation '{}') "
"added with {} submissions and {} replies".format(
codename, journalist_designation, num_submissions, num_replies))
def add_source() -> Tuple[Source, str]:
"""
Adds a single source.
"""
codename = PassphraseGenerator.get_default().generate_passphrase()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_designation = current_app.crypto_util.display_id()
source = Source(filesystem_id, journalist_designation)
source.pending = False
db.session.add(source)
db.session.commit()
# Create source directory in store
os.mkdir(current_app.storage.path(source.filesystem_id))
# Generate source key
current_app.crypto_util.genkeypair(source.filesystem_id, codename)
return source, codename
def init_source_without_keypair():
"""Initialize a source: create their database record and the
filesystem directory that stores their submissions & replies.
Return a source object and their codename string.
:returns: A 2-tuple. The first entry, the :class:`Source`
initialized. The second, their codename string.
"""
# Create source identity and database record
codename = PassphraseGenerator.get_default().generate_passphrase()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_filename = current_app.crypto_util.display_id()
source = Source(filesystem_id, journalist_filename)
db.session.add(source)
db.session.commit()
# Create the directory to store their submissions and replies
os.mkdir(current_app.storage.path(source.filesystem_id))
return source, codename
def process_results(source_list):
'''
function to process results and transform them to a list of objects
Args:
source_list:dictionary cotaining source details
Returns:
source_results: A list of source objects
'''
source_results = []
for source_item in source_list:
id = source_item.get('id')
name = source_item.get('name')
description = source_item.get('description')
url = source_item.get('url')
if id:
source_object = Source(id,name,description,url)
source_results.append(source_object)
return source_results
def upload():
uf = request.files['input-b1']
file_dir=os.path.join(basedir,app.config['UPLOAD_FOLDER'])
if not os.path.exists(file_dir):
os.makedirs(file_dir)
if uf and allowed_file(uf.filename):
# size = len(uf.read())
size = 1
if size<51200000:
filename = secure_filename(uf.filename)
# currentpath = os.path.abspath(os.path.dirname(__file__))
# ossep = os.path.sep
ext = filename.rsplit('.',1)[1]
name = filename.rsplit('.',1)[0]
unix_time = int(time.time())
new_filename=name + '_' + str(unix_time)+'.'+ext
# savepath = currentpath + ossep+'uploadfolder'+ ossep + filename
savepath = os.path.join(file_dir,new_filename)
# uf.save(savepath)
uf.save(savepath)
source = Source(path=savepath,filename=new_filename)
user_id = session.get('user_id')
user = User.query.filter(User.id == user_id).first()
source.owner = user
try:
db.session.add(source)
db.session.commit()
except Exception as reason:
s=str(reason)
list = re.split(r'[()]+', s)
flash(u'Upload Failed：(%s)(%%s)'%list[1]%list[3])
return redirect(url_for('index'))
else:
flash(u'Upload Succ')
return redirect(url_for('index'))
else:
flash(u'error:File size should less than 50M')
return redirect(url_for('index'))
else:
flash(u'error:File type should be xls or xlsx')
return redirect(url_for('index'))
def merge_work(dbsession, person, title, source):
work = dbsession.query(Work).filter(and_(Work.person_id == person.id, Work.title == title)).first()
if not work:
work = Work(person=person, title=title, status='unconfirmed')
dbsession.add(work)
dbsession.commit()
else:
work_source = dbsession.query(WorkSource).join(Source).filter(and_(WorkSource.work == work,
Source.url == source['url'])).first()
if not work_source:
db_source = dbsession.query(Source).filter(Source.url == source['url']).first()
if not db_source:
db_source = Source(label=source['label'], url=source['url'])
dbsession.add(db_source)
work_source = WorkSource(work=work, source=db_source, timestamp=source['timestamp'])
dbsession.add(work_source)
else:
work_source.timestamp = source['timestamp']
dbsession.add(work_source)
dbsession.commit()
return work
