def record_datafile_status(dataset_authority, dataset_identifier,
storage_identifier, status, createdate):
current_status = get_datafile_status(dataset_authority, dataset_identifier,
storage_identifier)
backup_db_connection = get_backupdb_connection()
cursor = backup_db_connection.cursor()
createdate_str = createdate.strftime('%Y-%m-%d %H:%M:%S.%f')
nowdate_str = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')
if current_status is None:
query = "INSERT INTO datafilestatus (status, createdate, lastbackuptime, lastbackupmethod, datasetidentifier, storageidentifier) VALUES (%s, %s, %s, %s, %s, %s);"
else:
query = "UPDATE datafilestatus SET status=%s, createdate=%s, lastbackuptime=%s, lastbackupmethod=%s WHERE datasetidentifier=%s AND storageidentifier=%s;"
dataset_id = dataset_authority + "/" + dataset_identifier
backup_method = ConfigSectionMap("Backup")['storagetype']
cursor.execute(query, (status, createdate_str, nowdate_str, backup_method,
dataset_id, storage_identifier))
# finalize transaction:
backup_db_connection.commit()
cursor.close()
def open_ssh_client():
ssh_host = ConfigSectionMap("Backup")['sshhost']
ssh_port = ConfigSectionMap("Backup")['sshport']
ssh_username = ConfigSectionMap("Backup")['sshusername']
print "SSH Host: %s" % (ssh_host)
print "SSH Port: %s" % (ssh_port)
print "SSH Username: %s" % (ssh_username)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=ssh_host, username=ssh_username)
print "Connected!"
return ssh_client
def open_storage_object_filesystem(dataset_authority, dataset_identifier,
object_location, is_tabular_data):
filesystem_directory = ConfigSectionMap(
"Repository")['filesystemdirectory']
if (is_tabular_data is not None):
object_location += ".orig"
file_path = filesystem_directory + "/" + dataset_authority + "/" + dataset_identifier + "/" + object_location
byte_stream = io.open(file_path, "rb")
return byte_stream
def backup_file_swift (file_input, dataset_authority, dataset_identifier, storage_identifier):
auth_url = ConfigSectionMap("Backup")['swiftauthurl']
auth_version = ConfigSectionMap("Backup")['swiftauthversion']
user = ConfigSectionMap("Backup")['swiftuser']
tenant = ConfigSectionMap("Backup")['swifttenant']
key = ConfigSectionMap("Backup")['swiftkey']
conn = swiftclient.Connection(
authurl=auth_url,
user=user,
key=key,
tenant_name=tenant,
auth_version=auth_version
)
container_name = dataset_authority + ":" + dataset_identifier
conn.put(container_name)
conn.put_object(container_name, storage_identifier, file_input)
def backup_file(file_input, dataset_authority, dataset_identifier,
storage_identifier, checksum_type, checksum_value, file_size):
storage_type = ConfigSectionMap("Backup")['storagetype']
if storage_type == 'swift':
#backup_file_swift(file_input, dataset_authority, dataset_identifier, storage_identifier, checksum_type, checksum_value, file_size)
raise NotImplementedError('no backup_swift yet')
elif storage_type == 'ssh':
backup_file_ssh(file_input, dataset_authority, dataset_identifier,
storage_identifier, checksum_type, checksum_value,
file_size)
else:
raise ValueError(
"only ssh/sftp and swift are supported as backup storage media")
def send_notification(text):
try:
notification_address = ConfigSectionMap("Notifications")['email']
except:
notification_address = None
if (notification_address is None):
raise ValueError('Notification email address is not configured')
nowdate_str = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M')
subject_str = ('Dataverse datafile backup report [%s]' % nowdate_str)
p = Popen(['mail','-s',subject_str,notification_address], stdout=PIPE, stdin=PIPE, stderr=PIPE)
stdout_data = p.communicate(input=text)[0]
def create_database_connection(database='database'):
Host = ConfigSectionMap("Database")['host']
Port = ConfigSectionMap("Database")['port']
Database = ConfigSectionMap("Database")[database]
Username = ConfigSectionMap("Database")['username']
Password = ConfigSectionMap("Database")['password']
#print "Database Host: %s" % (Host)
#print "Database Port: %s" % (Port)
#print "Database Name: %s" % (Database)
#print "Username: %s" % (Username)
#print "Password: %s" % (Password)
#Define our connection string
conn_string = "host='" + Host + "' dbname='" + Database + "' user='******' password='******'"
#print "Connecting to database\n->%s" % (conn_string)
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
#print "Connected!\n"
return conn
def transfer_file(local_flo, dataset_authority, dataset_identifier,
storage_identifier, byte_size):
sftp_client = my_ssh_client.open_sftp()
remote_dir = dataset_authority + "/" + dataset_identifier
subdirs = remote_dir.split("/")
cdir = ConfigSectionMap("Backup")['backupdirectory'] + "/"
for subdir in subdirs:
try:
cdir = cdir + subdir + "/"
sftpattr = sftp_client.stat(cdir)
except IOError:
#print "directory "+cdir+" does not exist (creating)"
sftp_client.mkdir(cdir)
#else:
# print "directory "+cdir+" already exists"
m = re.search('^([a-z0-9]*)://(.*)$', storage_identifier)
if m is not None:
storageTag = m.group(1)
storage_identifier = re.sub('^.*:', '', storage_identifier)
remote_file = cdir + storage_identifier
if (type(local_flo) is str):
sftp_client.put(local_flo, remote_file)
else:
# assume it's a stream:
# sftp_client.putfo() is convenient, but appears to be unavailable in older
# versions of paramiko; so we'll be using .read() and .write() instead:
#sftp_client.putfo(local_flo,remote_file,byte_size)
sftp_stream = sftp_client.open(remote_file, "wb")
while True:
buffer = local_flo.read(32 * 1024)
if len(buffer) == 0:
break
sftp_stream.write(buffer)
sftp_stream.close()
sftp_client.close()
print "File transfered."
return remote_file
from config import ConfigSectionMap
led_pin = int(ConfigSectionMap("dust")['led_pin'])
try:
import RPi.GPIO as GPIO
GPIO.setmode(GPIO.BCM)
led = True
GPIO.setup(led_pin, GPIO.OUT)
except ImportError:
led = False
def on():
if led is True:
GPIO.output(led_pin, GPIO.HIGH)
def off():
if led is True:
GPIO.output(led_pin, GPIO.LOW)
from config import ConfigSectionMap
input_pin = int(ConfigSectionMap("gpio")['input_pin'])
rs = int(ConfigSectionMap("lcd")['rs'])
en = int(ConfigSectionMap("lcd")['en'])
d4 = int(ConfigSectionMap("lcd")['d4'])
d5 = int(ConfigSectionMap("lcd")['d5'])
d6 = int(ConfigSectionMap("lcd")['d6'])
d7 = int(ConfigSectionMap("lcd")['d7'])
try:
import Adafruit_CharLCD as LCD
import RPi.GPIO as GPIO
except ImportError:
lcd = None
else:
lcd = LCD.Adafruit_CharLCD(rs, en, d4, d5, d6, d7, 16, 2)
GPIO.setmode(GPIO.BCM)
GPIO.setup(input_pin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
def GPIO_input():
if lcd is not None:
return GPIO.input(input_pin)
else:
return False
def message(msg):
if lcd is not None:
lcd.message(msg)
import datetime
import time
from config import ConfigSectionMap
from run import run, reset, button_pushed, lcd_ready
from utils.timer import int_time, timer_list
interval = int(ConfigSectionMap("run")['interval'])
duration = int(ConfigSectionMap("run")['duration'])
schedule_time = ConfigSectionMap("schedule")['time']
timers = []
start_time = int_time()
elapsed_time = 0
schedule_toggle = False
def check_schedule(time_str):
global schedule_toggle
if schedule_toggle is True:
return False
scheduled_time = time.strptime(time_str, "%H:%M")
current_time = datetime.datetime.now()
if scheduled_time.tm_hour == current_time.hour and scheduled_time.tm_min == current_time.minute:
return True
else:
return False
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import re
from config import ConfigSectionMap
route_id = ConfigSectionMap("bus")['route_id']
def parse_time_string(time_string):
"""
parse time string into arrival object
:param time_string: "12분33초후[2번째 전]"
:return: list (minutes, seconds, stops)
"""
pattern = re.compile(
'(?:(?P<min>\d+)분|)(?:(?P<sec>\d+)초|)후\[(?P<stops>\d+)번째 전\]')
match = pattern.search(time_string)
s = None
if match is not None:
m = match.group('min')
s = match.group('sec')
stops = match.group('stops') # digit
if m is None: # "33초후[2번째 전]"
m = 0
if s is None: # "12분후[10번째 전]
s = 0
s = (int(m), int(s), int(stops))
from bus.bus_query import create_bus_qs
from utils.url import make_url
import grequests
from config import ConfigSectionMap
arrival_api_url = 'http://ws.bus.go.kr/api/rest/stationinfo/getStationByUid'
key = ConfigSectionMap("api_key")['bus']
station_id = ConfigSectionMap("bus")['station_id']
def bus_grequests(url, key, station_id):
bus_qs = create_bus_qs(key, station_id)
return grequests.get(make_url(url, bus_qs))
def bus_request():
return bus_grequests(arrival_api_url, key, station_id)
from bs4 import BeautifulSoup
import uuid
import json
import os
import boto3
from boto3.s3.transfer import S3Transfer
import requests
import wget
import ssl
from functools import wraps
from config import ConfigSectionMap
from houzzCrawler import Data
cfg = ConfigSectionMap('aws')
def sslwrap(func):
@wraps(func)
def bar(*args, **kw):
kw['ssl_version'] = ssl.PROTOCOL_TLSv1
return func(*args, **kw)
return bar
ssl.wrap_socket = sslwrap(ssl.wrap_socket)
client = boto3.client('s3',
aws_access_key_id=cfg['aws_access_key_id'],
aws_secret_access_key=cfg['aws_secret_access_key'])
import datetime
from xml.etree import ElementTree
import grequests
from bus.bus_request import bus_request
from bus.bus_view import arrivals_from_xml, bus_arrival_view
from dust.dust_request import dust_request
from dust.dust_view import dust_from_xml, dust_view
from gpio import lcd
from gpio import led
from weather.weather_request import live_weather_request, forecast_weather_request
from weather.weather_view import weather_live_from_json, weather_forecast_from_json, weather_view
from config import ConfigSectionMap
pm10_threshold = int(ConfigSectionMap("dust")['pm10_threshold'])
pm25_threshold = int(ConfigSectionMap("dust")['pm25_threshold'])
def run():
lcd.clear()
lcd.message("Loading...")
weather_live_req = live_weather_request()
weather_forecast_req = forecast_weather_request()
bus_arrival_req = bus_request()
dust_req = dust_request()
grequests.map(
(weather_live_req, weather_forecast_req, bus_arrival_req, dust_req)
)
from dust.dust_query import create_dust_qs
from utils.url import make_url
import grequests
from config import ConfigSectionMap
dust_api_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getMsrstnAcctoRltmMesureDnsty'
key = ConfigSectionMap("api_key")['dust']
station_name = ConfigSectionMap("dust")['station_name']
def dust_grequests(url, key, station_id):
dust_qs = create_dust_qs(key, station_id)
return grequests.get(make_url(url, dust_qs))
def dust_request():
return dust_grequests(dust_api_url, key, station_name)
from flask import Flask, flash, request, jsonify, render_template, redirect
import json
import os
import sys
import urllib.request
from werkzeug.utils import secure_filename
from flask_dropzone import Dropzone
#Modulos Internos
from modules.lotes_site.lotesite import getlotesite
from modules.detail_lote_site.detaillotesite import getdetaillotesite
from app import UPLOAD_FOLDER
from config import ConfigSectionMap
#Configuraciones Iiciales
extension = ConfigSectionMap('Extension')
separator = ConfigSectionMap('Separator')
fileencode = ConfigSectionMap('FileEncode')
EXTENSION = extension['extensionallow']
EXTENSION = (json.loads(EXTENSION.replace("'", '"')))
SEPARATOR = separator['type']
SEPARATOR = (json.loads(SEPARATOR.replace("'", '"')))
FILEENCODE = fileencode['typefile']
FILEENCODE = (json.loads(FILEENCODE.replace("'", '"')))
app = Flask(__name__)
dropzone = Dropzone(app)
UPLOAD_FOLDER = '/home/quintanada/Pictures/upload/'
app.secret_key = "secret key"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
from utils.url import make_url
from config import ConfigSectionMap
from weather.weather_query import create_weather_live_qs, create_weather_forecast_qs
import grequests
import datetime
forecast_api_url = 'http://newsky2.kma.go.kr/service/SecndSrtpdFrcstInfoService2/ForecastSpaceData'
live_api_url = 'http://newsky2.kma.go.kr/service/SecndSrtpdFrcstInfoService2/ForecastGrib'
nx = int(ConfigSectionMap("weather")['nx'])
ny = int(ConfigSectionMap("weather")['ny'])
live_basetime_delta = int(ConfigSectionMap("weather_basetime_delta")['live'])
forecast_basetime_delta = int(ConfigSectionMap("weather_basetime_delta")['forecast'])
key = ConfigSectionMap("api_key")['weather']
def live_weather_request():
now = datetime.datetime.now()
qs = create_weather_live_qs(key, nx, ny, now, live_basetime_delta)
return grequests.get(make_url(live_api_url, qs))
def forecast_weather_request():
now = datetime.datetime.now()
qs = create_weather_forecast_qs(key, nx, ny, now, forecast_basetime_delta)
return grequests.get(make_url(forecast_api_url, qs))
import requests
from bs4 import BeautifulSoup
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
import ssl
from functools import wraps
from config import ConfigSectionMap
cfg = ConfigSectionMap('setup')
START = cfg['start']
END = cfg['end']
session = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
session.proxies = proxies
session.headers = headers
session.mount('https://', HTTPAdapter(max_retries=retries))
def sslwrap(func):
@wraps(func)
def bar(*args, **kw):
kw['ssl_version'] = ssl.PROTOCOL_TLSv1
return func(*args, **kw)
return bar
ssl.wrap_socket = sslwrap(ssl.wrap_socket)
import requests
from config import ConfigSectionMap
from bs4 import BeautifulSoup
import boto3
from boto3.s3.transfer import S3Transfer
import wget
import os
import json
import uuid
cfg_setup = ConfigSectionMap('setup')
class Profile:
def __init__(self, soup):
self.soup = soup
self.name = self.extract_name(self.soup)
self.user_name = self.extract_username(self.soup)
self.profile_pic = self.extract_profile_pic(self.soup)
self.description = self.extract_description(self.soup)
self.services_provided = self.extract_services_provided(self.soup)
self.areas_serviced = self.extract_areas_serviced(self.soup)
self.social_details = self.extract_social_details(self.soup)
self.profession = self.extract_profession(self.soup)
self.contact_details = self.extract_contact_details(self.soup)
self.followers = self.extract_followers(self.soup)
def __repr__(self):
return repr(
json.dumps({
"name": self.name,
