def fetch_monthly(self):
        month_ago = (datetime.datetime.now() - datetime.timedelta(days=30))
        unixtime_month_ago = month_ago.strftime('%s')
        start = datetime.datetime.now()
        print(f"Last searched date: {month_ago}")

        running = True
        #           Since this API responds with jsons of a maximum size of 1000 images, we need to
        #           make several requests to gather images from entire month; with every request
        #           I check the last image in it and start from that date on the next request, until
        #           I have all images necessary.
        while running:
            # range as in FROM later TO earlier, dates are in UnixTime
            url = (
                f'http://api.deckchair.com/v1/camera/{self.cam_id}'
                f'/images?from={unixtime_month_ago}&to={start.strftime("%s")}')
            cam_imgs = request_to_json(url)

            start = convert_to_datetime(cam_imgs[0]['taken'])
            end = convert_to_datetime(cam_imgs[-1]['taken'])

            print(f"{start} - {end}", url, f"({len(cam_imgs)} images found)")

            for img in cam_imgs:
                img_date = convert_to_datetime(img['taken'])
                if img_date <= month_ago:
                    running = False
                    break
                else:
                    img_url = f'http://api.deckchair.com/v1/viewer/image/{img["_id"]}'
                    yield img_date, requests.get(img_url).content

            start = end
    def fetch_latest(self):
        cam_imgs = request_to_json(
            f'http://api.deckchair.com/v1/camera/{self.cam_id}/images')

        img_id = cam_imgs[0]['_id']
        img_date = cam_imgs[0]['taken']
        print(datetime.datetime.now(), convert_to_datetime(img_date))
        url = f'http://api.deckchair.com/v1/viewer/image/{img_id}'
        return requests.get(url).content, convert_to_datetime(img_date)
Пример #3
0
def build_daily_trading():

    sql_create_table = '''
    DROP TABLE IF EXISTS daily_price;
    CREATE TABLE daily_price (
    id INT PRIMARY KEY AUTO_INCREMENT,
    ticker_id INT NOT NULL,
    ticker VARCHAR(32),
    date_time datetime,
    open_price double,
    high_price double,
    low_price double,
    close_price double,
    volume double,
    amount double,
    free_turnover_ratio double,
    trade_status varchar(32),
    maxup_or_maxdown int,
    last_updated_datetime datetime)engine = InnoDB
    '''
    cur.execute(sql_create_table)

    sql_select_ticker_id = '''
    SELECT id,ticker,ipo_date FROM symbols 
    '''
    cur.execute(sql_select_ticker_id)
    data = cur.fetchall()

    column_str = '''
    open_price,high_price,low_price,close_price,volume,amount,
    free_turnover_ratio,trade_status,maxup_or_maxdown,ticker,
    date_time,ticker_id,last_updated_datetime
    '''
    insert_str = ('%s' * 13)[:-1]
    final_str = '''
    INSERT INTO daily_price (%s) VALUES (%s)
    ''' % (column_str, insert_str)

    total_length = len(data)
    i = 0
    for idx, ticker, ipo_date in data:
        i += 1
        now = dt.datetime.now()
        start_date = ipo_date.strftime('%Y%m%d')
        end_date = now.strftime('%Y%m%d')
        last_updated_datetime = now
        ticker_data = get_stock_daily_price(ticker, start_date, end_date)
        ticker_data['ticker_id'] = idx
        ticker_data['last_updated_datetime'] = now
        ticker_data['date_time'] = convert_to_datetime(
            ticker_data['date_time'])
        ticker_data['last_updated_datetime'] = convert_to_datetime(
            ticker_data['last_updated_datetime'])
        ticker_data = ticker_data.values.tolist()
        cur.executemany(final_str, ticker_data)
        print '%s 股票已经成功插入数据库' % ticker
        print '%s/%s' % (i, total_length)
Пример #4
0
def get_reverse_trade_price(order, binance_client):
    order_time = int(order['updateTime'])
    start_date = str(convert_to_datetime(order_time).date())
    end_date = str(convert_to_datetime(order_time).date())
    # To patch this to take in price_df from s3 bucket or github repository
    moment_price_df = pd.DataFrame(binance_client.get_historical_klines(
        "BTCUSDT", binance_client.KLINE_INTERVAL_30MINUTE, start_date, end_date))  # [Potential Bug]
    closest_price = moment_price_df[moment_price_df[0]
                                    < order_time].iloc[-1, 4]
    return closest_price
Пример #5
0
    def object_from_dict(cls, data):
        user = User(name=data['name'])

        fields = ['type', 'bio', 'logo', 'display_name']

        for field in fields:
            setattr(user, field, data[field])

        user.id = data['_id']
        user.created__at = convert_to_datetime(data['created_at'])
        user.updated__at = convert_to_datetime(data['updated_at'])

        return user
Пример #6
0
    def object_from_dict(cls, data):
        team = Team(name=data['name'])

        fields = ['info', 'display_name', 'logo', 'banner', 'background']

        for field in fields:
            setattr(team, field, data[field])

        team.id = data['_id']
        team.created__at = convert_to_datetime(data['created_at'])
        team.updated__at = convert_to_datetime(data['updated_at'])

        return team
Пример #7
0
    def __init__(self, elem_lst):
        self.app_no = elem_lst[0]  #工单受理唯一标识号
        self.handle_id = elem_lst[1]  #工单处理编号
        self.comm_no = elem_lst[2]  #通讯号码

        self.req_begin_date = None  #申请的提交时间
        self.req_begin_date = utils.convert_to_datetime(
            '01_S_COMM_REC.TSV', elem_lst[3])

        self.req_finish_date = None  #服务请求结束时间
        self.req_finish_date = utils.convert_to_datetime(
            '01_S_COMM_REC.TSV', elem_lst[4])

        self.org_no = elem_lst[5]  #供电单位
        self.busi_type_code = elem_lst[6]  #受理业务类型
        self.wkst_busi_type_code = elem_lst[7]  #工单业务类型
Пример #8
0
    def object_from_dict(cls, data):
        channel = Channel(name=data['name'])
        fields = [
            'mature', 'status', 'broadcaster_language', 'display_name',
            'game', 'delay', 'language', 'logo', 'banner', 'video_banner',
            'background', 'profile_banner', 'profile_banner_background_color',
            'partner', 'url', 'views', 'followers'
        ]
        for field in fields:
            setattr(channel, field, data[field])

        channel.id = data['_id']
        channel.created_at = convert_to_datetime(data['created_at'])
        channel.updated_at = convert_to_datetime(data['updated_at'])

        return channel
Пример #9
0
def print_episode(text, episode, lang, type_episode):
    if episode:
        date = convert_to_datetime(episode['airstamp'])
        episode['airtime'] = date.strftime("%H:%M")
        if lang == 'es':
            date = convert_to_timezone(date)
            episode['airdate'] = date.strftime("%d-%m-%Y")
            episode['airtime'] = date.strftime("%H:%M")
        text += translations[type_episode][lang].format(**episode)
        if episode['summary']:
            summary = remove_tag(episode['summary'])
            if lang != 'en':
                try:
                    summary = config.gs.translate(summary, lang)
                except urllib.exceptions.HTTPError:
                    pass
            text += summary
        if type_episode == 'next_episode':
            date_str = episode['airdate'] + ' ' + episode['airtime']
            date_time = datetime.datetime.strptime(date_str, "%d-%m-%Y %H:%M")
            delta = date_time - datetime.datetime.now()
            if delta.days:
                text += translations['left'][lang].format(days=delta.days)
        text += '\n \n'
    return text
Пример #10
0
    def __init__(self, elem_lst):
        self.app_no = elem_lst[0]  #工单受理唯一标识号
        self._id = elem_lst[1]  #工单唯一标识
        self.busi_type_code = elem_lst[
            2]  #业务类型编码 "010 003 001 007 015 005 018 009 008 006"
        self.urban_rural_flag = elem_lst[3]  #城乡类别标识
        self.org_no = elem_lst[4]  #供电单位编码

        self.handle_time = None  #工单受理时间
        self.handle_time = utils.convert_to_datetime(
            "00_ARC_S_95598_WKST_*.TSV", elem_lst[5])

        self.accept_content = None  #工单受理内容
        self.accept_content = utils.gbk2utf(elem_lst[6])

        self.handle_opinion = elem_lst[7]  #受理意见
        self.calling_no = elem_lst[8]  #主叫号码

        self.elec_type = elem_lst[9]  #用电类别
        if len(elem_lst[9]) == 0:
            self.elec_type = "null"

        self.cust_no = elem_lst[10]  #客户编号 (!!!!结果提交字段)
        self.prov_org_no = elem_lst[11]  #所属省(市)公司供电单位编码
        self.city_org_no = elem_lst[12]  #所属市(区)公司供电单位编码

        self.poweroff_ids = []  #从content中抽取poweroffid
Пример #11
0
def build_trading_days():
    '''
    '''
    today = dt.date.today().strftime('%Y-%m-%d')
    start_date = '1991-01-01'
    end_date = today
    last_updated_date = end_date

    sql_create_table = '''
    DROP TABLE IF EXISTS tradedates;
    CREATE TABLE tradedates (
    id INT PRIMARY KEY AUTO_INCREMENT,
    date_time datetime,
    last_updated_date datetime
    )engine = InnoDB
    '''
    cur.execute(sql_create_table)

    trading_days = get_trading_days(start_date, end_date).tolist()
    trading_days = list(convert_to_datetime(trading_days))

    schema_list = []
    for each in trading_days:
        schema_list.append([each, last_updated_date])

    column_str = 'date_time,last_updated_date'
    insert_str = ('%s,' * 2)[:-1]
    final_str = '''
    INSERT INTO tradedates (%s) VALUES (%s)
    ''' % (column_str, insert_str)

    cur.executemany(final_str, schema_list)
    con.commit()
Пример #12
0
    def object_from_dict(cls, data):
        follow = Follow()

        follow.notifications = data['notifications']
        follow.created_at = convert_to_datetime(data['created_at'])

        follow.user = User.object_from_dict(data['user'])

        return follow
Пример #13
0
    def fetch_todays(self):
        today = datetime.date.today()
        print(today)
        url = f'http://api.deckchair.com/v1/camera/{self.cam_id}/images?from={today.strftime("%s")}'

        cam_imgs = request_to_json(url)

        for img in cam_imgs:
            # print(img['taken'])

            url = f'http://api.deckchair.com/v1/viewer/image/{img["_id"]}'
            yield convert_to_datetime(img['taken']), requests.get(url).content
Пример #14
0
def read_tasks(file_to_read):
	tasks=None
	try:
		with open(file_to_read, "r") as f:
			tasks = json.load(f)
	except IOError:
		print(f"Couldn't read {file_to_read} or {file_to_read} does not exsist!!")
	except Exception as e:
		print("EXCEPTION OCCURED!!!")
		print(e)

	return sorted(convert_to_datetime(tasks), key=lambda d : d["start"])
Пример #15
0
    def __init__(self, elem_lst):
        self.cons_no = elem_lst[0]  #用户编号
        self.org_no = elem_lst[1]  #供电单位代码

        self.charge_ym = None  #收费年月
        self.charge_ym = utils.convert_to_date_YM('12_A_PAY_FLOW.TSV',
                                                  elem_lst[2])

        self.charge_date = None  #收费日期
        self.charge_date = utils.convert_to_datetime('12_A_PAY_FLOW.TSV',
                                                     elem_lst[3])

        self.pay_mode = utils.convert_to_int(elem_lst[4])  #缴费方式
Пример #16
0
    def __init__(self, elem_lst):
        self.poweroff_id = elem_lst[0]  #停电信息唯一标识号
        self.app_no = elem_lst[1]  #申请编号

        self.start_time = None  #停电开始时间
        self.start_time = utils.convert_to_datetime('02_S_REGION_OUTRAGE.TSV',
                                                    elem_lst[3])

        self.stop_date = None  #停电结束时间
        self.stop_date = utils.convert_to_datetime('02_S_REGION_OUTRAGE.TSV',
                                                   elem_lst[4])

        self.org_no = elem_lst[5]  #停电所属供电单位
        self.type_code = elem_lst[6]  #停电类型
        self.poweroff_reason = elem_lst[8]  #停电原因

        self.power_time = None  #现场送电时间
        self.power_time = utils.convert_to_datetime('02_S_REGION_OUTRAGE.TSV',
                                                    elem_lst[31])

        self.poweroff_area = elem_lst[7]  #停电范围
        self.poweroff_scope = elem_lst[2]  #停电区域
Пример #17
0
    def object_from_dict(cls, data):
        video = Video(title=data['title'])

        fields = [
            'status', 'tag_list', 'description', 'title', 'url', 'views',
            'length', 'game', 'broadcast_id', 'broadcast_type', 'preview',
            'resolutions', 'fps'
        ]

        for field in fields:
            setattr(video, field, data[field])

        video.id = data['_id']
        video.recorded_at = convert_to_datetime(data['recorded_at'])

        return video
Пример #18
0
    def __init__(self, elem_lst):
        self.app_no = elem_lst[0]  #工单受理唯一标识号

        self.oversee_time = None  #催办督办时间
        self.oversee_time = utils.convert_to_datetime('03_S_INFO_OVERSEE.TSV',
                                                      elem_lst[1])

        self.cust_no = elem_lst[2]  #客户编号
        self.cust_name = elem_lst[3]  #来电客户姓名

        self.oversee_reason = None  #催办督办原因
        self.oversee_reason = utils.gbk2utf(elem_lst[4])
        self.oversee_content = None  #催办督办内容
        self.oversee_content = utils.gbk2utf(elem_lst[5])

        self.oversee_app_no = elem_lst[6]  #被催办督办工单编号
        self.org_or_dept = elem_lst[7]  #被催办督办单位或部门
        self.app_busi_type_code = elem_lst[8]  #被催办工单类型
        self.org_no = elem_lst[9]  #供电单位
Пример #19
0
def update():
    try:
        db = utils.load_pickle(DB_PATH)
        last_update = sorted(db['date'])[-1]
    except:
        utils.download(DB_URL, DB_PATH)
        db = utils.load_pickle(DB_PATH)
        last_update = sorted(db['date'])[-1]

    # query arxiv api
    n_added = 0
    indx = 0

    while indx < MAX_ITER:

        url = BASE_URL + QUERY_FMT.format(DEF_QUERY, indx, RESULTS_PER_ITER)
        try:
            with urllib.request.urlopen(url, timeout=5.0) as url:
                response = url.read()
        except TimeoutError:
            continue
        response = feedparser.parse(response)

        for entry in response.entries:
            e = utils.encode_feedparser_dict(entry)
            paper_url = utils.parse_arxiv_url(e["link"])
            date = e["published"]
            date = utils.convert_to_datetime(date)

            # content already in database
            if paper_url in db["url"]:
                if date <= last_update:
                    indx = MAX_ITER
                    break
                else:
                    continue

            # retrieve and clean some text
            title = e["title"]
            title = utils.rem_tex_fmt(title)
            authors = ", ".join(f"{n['name']}" for n in e["authors"])
            abstract = e["summary"]
            abstract = utils.rem_tex_fmt(abstract)
            other_urls = utils.extract_url(abstract)
            journal = e["arxiv_journal_ref"] if "arxiv_journal_ref" in e else ""
            journal = utils.rem_tex_fmt(journal)

            db["date"].append(date)
            db["url"].append(paper_url)
            db["title"].append(title)
            db["authors"].append(authors)
            db["abstract"].append(abstract)
            db["journal"].append(journal)
            db["other_urls"].append(other_urls)
            n_added += 1

        if len(response.entries) == 0:
            utils.progress_bar(indx / MAX_ITER,
                               status="API not responding. retrying...")
        if indx == MAX_ITER:
            utils.progress_bar(1)
        else:
            indx += 100
            utils.progress_bar(indx / MAX_ITER,
                               status=f"Fetching papers from {date}...")
        time.sleep(WAIT_TIME)
    print(f"{n_added} papers added to database")

    if True:
        indx = list(np.argsort(db["date"]))
        db["date"] = list(np.array(db["date"])[indx])
        db["url"] = list(np.array(db["url"])[indx])
        db["title"] = list(np.array(db["title"])[indx])
        db["authors"] = list(np.array(db["authors"])[indx])
        db["abstract"] = list(np.array(db["abstract"])[indx])
        db["journal"] = list(np.array(db["journal"])[indx])
        db["other_urls"] = list(np.array(db["other_urls"])[indx])
        utils.save_pickle(DB_PATH, db)

        tkn_corpus = []
        for indx in range(len(db["url"])):
            title = db["title"][indx].lower()
            abstract = utils.filter_abstract(db["abstract"][indx].lower())
            tkn_corpus.append((title + " " + abstract).split(" "))
        bm25 = BM25Okapi(tkn_corpus)
        utils.save_pickle(CACHE_BM25, bm25)
Пример #20
0
def date_to_timestamp(date):
    return utils.convert_to_datetime(date)
Пример #21
0
 def test_convert_to_datetime(self):
     expected = datetime.datetime(2011, 11, 11, 0, 0)
     actual = convert_to_datetime('11/11/2011')
     self.assertEqual(actual, expected)