def get(self, channel_id, video_id):
data_source.view_video(channel_id, video_id)
video = data_source.get_video(channel_id, video_id)
if video:
self.render_dict_as_json(video)
else:
self.response.out.write("{}")
def get(self):
channel_id = self.request.get("channel_id")
values = {}
if channel_id:
offset, limit = handlers.parse_offset_and_limit(self)
videos = data_source.get_videos_model_in_channel(channel_id, offset, limit)
channel = data_source.get_channel(channel_id)
if channel:
values["channel"] = channel
# If delete is set we will delete all the channel as well as its videos.
delete = self.request.get("delete")
if delete == "true":
q = VideoModel.all()
q.ancestor(channel)
for video in q:
video.delete()
channel.delete()
self.redirect(router_path["admin_channel_list"])
return
if videos:
values["videos"] = videos
values["offset"] = offset
values["limit"] = limit
self.render("ChannelUpdate.html", values)
def workspace_test():
print_data('workspaces objects', br=False)
for index in range(3):
w = Workspace()
w.name = 'New workspace name'
w.description = 'Some new description'
w.save()
workspaces = Workspace.all()
print_data('new objects -> model.all()', workspaces)
w.name = 'Updated name'
w.save()
workspaces = Workspace.all()
print_data('UPDATED -> model.all()', workspaces)
workspaces = Workspace.get(id=w.id, name=w.name)
print_data('GET -> model.get()', [workspaces])
workspaces = Workspace.filter(name='New workspace name')
print_data('FILTER -> model.filter()', workspaces)
for index in range(2):
o = Application()
o.workspace_id = w.guid
o.save()
a = View()
a.application_id = o.guid
a.save()
a = Resource()
a.application_id = o.guid
a.save()
for index in range(3):
o = Widget()
o.workspace_id = w.guid
o.save()
for index in range(3):
o = DataSource()
o.workspace_id = w.guid
o.save()
objects = Workspace.all() + Resource.all() + Application.all() + Widget.all() + DataSource.all() + View.all()
print_data('All objects in db', objects)
# [w.delete() for w in Workspace.all()]
workspaces = Workspace.all()
print_data('cleaned', workspaces)
workspaces = Workspace.filter(include_deleted=True)
print_data('cleaned with deleted if exists', workspaces)
objects = Workspace.all() + Resource.all() + Application.all() + Widget.all() + DataSource.all() + View.all()
print_data('no objects left', objects)
def post(self):
channel_id = self.request.get("channel_id")
video_id = int(self.request.get("video_id"))
video = data_source.get_video(channel_id, video_id)
key = self.user
if not key: key = self.request.remote_addr
if video:
data_source.dislike_video(key, channel_id, video_id)
video["dislike"] += 1
self.render_dict_as_json(video)
def get(self, channel_id, video_id = '', comment_id = ''):
channel = data_source.get_channel(channel_id)
offset, limit = handlers.parse_offset_and_limit(self)
videos = data_source.get_videos_in_channel(channel_id, offset = offset, limit = limit)
ret = {}
ret["channel"] = channel
ret["videos"] = videos
ret["offset"] = offset
ret["limit"] = limit
self.render("ChannelListPage.html", ret)
def post(self):
channel_id = self.request.get("channel_id")
logging.info("Persisting data in channel: %s" % channel_id)
videos = data_source.get_videos_in_channel(channel_id, 0, data_source.MAX_CHANNEL_SIZE)
logging.info("Got videos: %d" % len(videos))
for v in videos:
video_model = data_source.get_video_model(channel_id, v["video_id"])
video_model.final_score = v["final_score"]
logging.info("Updating video %d with final_score %f" % (v["video_id"], v["final_score"]))
video_model.put()
def post(self):
logging.info("Start to execute a parse task.")
name = self.request.get("name")
if name == "youku_girls":
from tools import youku_parser
video_infos = youku_parser.parse_url("http://www.youku.com/v_showlist/t2c86g0d1.html")
for video_info in video_infos:
if not data_source.get_video_by_external_id(video_info["source"], video_info["external_id"]):
video = data_source.create_video_from_dict("girls", video_info)
video.calculate_score()
video.put()
def post(self):
channel_id = self.request.get("channel_id")
video_id = self.request.get("video_id")
video = data_source.get_video(channel_id, video_id)
key = self.user
if not key: key = self.request.remote_addr
if video:
data_source.like_video(key, channel_id, video_id)
video["like"] += 1
self.render_dict_as_json(video)
else:
self.render_dict_as_json({"error" : "Video not found channel_id=%s, video_id=%s" % (channel_id, video_id)})
def get(self, channel_id):
channel = data_source.get_channel(channel_id)
if not channel:
self.render_dict_as_json({"error" : "Channel not found. channel_id=%s" % channel_id})
return
offset, limit = handlers.parse_offset_and_limit(self)
videos = data_source.get_videos_in_channel(channel_id, offset = offset, limit = limit)
ret = {}
ret["channel"] = channel.to_dict()
ret["videos"] = videos
ret["offset"] = offset
ret["limit"] = limit
self.render_dict_as_json(ret)
def get(self):
channel_id = self.request.get("channel_id")
video_id = self.request.get("video_id")
video = data_source.get_video(channel_id, video_id)
values = {}
if channel_id:
values["channel_id"] = channel_id
if video:
values["video"] = video
self.render("VideoUpdate.html", values)
def post(self):
channel_id = self.request.get("channel_id")
title = self.request.get("title")
cover_img = self.request.get("cover_img")
channel = data_source.get_channel(channel_id)
if channel:
self.response.out.write("exist")
else:
channel = ChannelModel(key_name = channel_id, title = title, cover_img = cover_img)
channel.put()
self.redirect(router_path["admin_channel_list"])
def make_data_source(name, func, func_lang, description="", dependencies=[]):
data_source = DataSource(
name=name,
description=description,
dependencies=dependencies,
transform_function=func,
transform_function_language=func_lang
)
db.session.add(data_source)
db.session.commit()
return data_source
def test_MySQL():
ds = DataSource(db_type="MySQL",
host="10.10.8.104",
database="fdm_db",
port=4000,
user="******",
password="******")
conn = get_Connection(ds)
sql = 'select spbm,spmc from fdm_db.f_dm_gy_spbm'
rows = conn.query(sql)
for row in rows:
print("spbm:%s, spmc:%s" % (row[0], row[1]))
def test_Impala():
ds = DataSource(db_type="Impala",
host="10.10.8.102",
database="fdm_db",
port=21050)
conn = get_Connection(ds)
sql = 'select zsxm_dm,zsxmmc,zsxmjc,xybz,yxbz,sjzsxm_dm,yxqz,yxqq from odm_db.o_hx_qg_dm_gy_zsxm'
rows = conn.query(sql)
for row in rows:
print(
"zsxm_dm:%s, zsxmmc:%s, zsxmjc:%s, xybz:%s, yxbz:%s, sjzsxm_dm:%s, yxqz:%s, yxqq:%s"
% (row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7]))
def add_data_source(uri):
db = connect()
matching = db.query(DataSource).filter_by(uri=uri).first()
if matching is None:
source = DataSource(uri=uri,
last_accessed=datetime.date.today(),
times_used=1)
db.add(source)
else:
matching.last_accessed = datetime.date.today()
matching.times_used += 1
db.commit()
def post(self):
channel_id = self.request.get("channel_id")
video_id = self.request.get("video_id")
channel = data_source.get_channel(channel_id)
video = data_source.get_video(channel_id, video_id)
if channel:
title = self.request.get("title")
cover_img = self.request.get("cover_img")
video_url = self.request.get("video_url")
editor_score = int(self.request.get("editor_score"))
if not video:
video = VideoModel(parent = channel, title = title, cover_img = cover_img, video_url = video_url, editor_score = editor_score)
else:
video.title = title
video.cover_img = cover_img
video.video_url = video_url
video.editor_score = editor_score
video.calculate_score()
video.put()
self.redirect(router_path["admin_channel_update"] + "?channel_id=%s" % (channel_id))
return self.get()
else:
self.response.out.write("channel not exist")
from models import DataSource
data_source_id = request.arguments.get('data_source_id', '')
command = request.arguments.get('command', 'update')
data_source = DataSource.get(guid=data_source_id)
if data_source:
disabled = '2' if command == 'delete' else '0'
form = self.dialog_update.form_update
form.connector.value = data_source.connector
form.connector.mode = disabled
form.workspace_id.value = data_source.workspace_id
form.data_source_id.value = data_source.guid
form.command.value = command
self.dialog_update.form_update.btn_update.label = command.title()
form.btn_update.action("setLabel", [command.title()])
form.btn_update.action(
"setClass",
['btn btn-danger' if command == 'delete' else 'btn btn-success'])
self.dialog_update.title = 'Data Source {}'.format(command.title())
self.dialog_update.show = '1'
def get(self, channel_id, video_id, comment_id=''):
data_source.view_video(channel_id, video_id)
video = data_source.get_video(channel_id, video_id)
values = {};
values["video"] = video;
self.render("ChannelListPage.html", values)
"""
GOOGLE API
"""
GOOGLE_API_GEOCODE_COORD_URL = "https://maps.googleapis.com/maps/api/geocode/json?latlng=%s,%s&sensor=false&key=%s"
GOOGLE_API_GEOCODE_ADDR_URL = "https://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false&key=%s"
GOOGLE_API_KEY = environ.get("GOOGLE_API_KEY")
"""
US PREPROCESSOR
- US_PROCESSOR_COLUMN_DEFINITIONS: The columns in the data files is inconsistent. This defines the fields for each file
- US_PROCESSOR_DATA_SOURCES: series component and file location tuple
"""
US_PROCESSOR_FILTERED_KEYS = set(["united_states"])
US_PROCESSOR_COLUMN_DEFINITIONS = {'confirmed': {'name': 5, 'key': 10, 'data': 11}, 'deaths': {'name': 5, 'key': 10, 'data': 12}}
US_PROCESSOR_DATA_SOURCES = [
DataSource("covid19", "confirmed", join(GITHUB_DIRECTORY, "CSSEGISandData/confirmed_us.csv")),
DataSource("covid19", "deaths", join(GITHUB_DIRECTORY, "CSSEGISandData/deaths_us.csv"))
]
"""
MX PREPROCESSOR
"""
MX_PROCESSOR_DATA_START_DATE = date(2020, 1, 23)
MX_PROCESSOR_DATA_FILE = join(GITHUB_DIRECTORY, 'carranco-sga/mx_data.csv')
MX_PROCESSOR_DATA_SOURCES = [
DataSource("covid19", "confirmed", join(OUTPUT_DIRECTORY, "confirmed_mx.csv")),
DataSource("covid19", "deaths", join(OUTPUT_DIRECTORY, "deaths_mx.csv"))
]
MX_PROCESSOR_COLUMN_SUFFIXES = {
'confirmed': '',
'deaths': '_D'
raise Exception('Workspace ID is not provided')
workspace_id = request.arguments.get('workspace_id')
command = request.arguments.get('command', u'')
workspace = Workspace.get(guid=workspace_id)
if not workspace:
self.action('goTo', ['/main'])
elif command in ['delete', 'update']:
if 'data_source_id' not in request.arguments:
raise Exception(u'DataSource ID is not provided')
data_source_id = request.arguments['data_source_id']
data_source = DataSource.get(guid=data_source_id,
workspace_id=workspace.guid)
if data_source:
if command == 'delete':
data_source.delete()
else:
connector = request.arguments['connector']
if connector:
data_source.connector = connector
data_source.save()
self.dialog_update.action('hide', ['0'])
elif command == 'create':
connector = request.arguments['connector']
if connector:
from app import create_app
from database import db
from models import DataSource, DataRange, Data
app = create_app()
with app.app_context():
db.init_app(app)
with app.app_context():
DataSource.query.all()
dsd = {}
dsd["name"] = "test"
dsd["description"] = ""
dsd["dependencies"] = []
dsd["transform_function"] = 'return [{"timestamp": "2017-10-05T14:48:00.000Z", "value": "1"}]'
dsd["transform_function_language"] = 'python'
ds = DataSource(**dsd)
with app.app_context():
db.session.add(ds)
db.session.commit()
with app.app_context():
print(DataSource.query.one().dependencies)
f = """
def foo():
return 5+5
print(foo())
"""
exec(f, {'__builtins__':{}, 'print': print})