def record_datafile_status(dataset_authority, dataset_identifier, storage_identifier, status, createdate): current_status = get_datafile_status(dataset_authority, dataset_identifier, storage_identifier) backup_db_connection = get_backupdb_connection() cursor = backup_db_connection.cursor() createdate_str = createdate.strftime('%Y-%m-%d %H:%M:%S.%f') nowdate_str = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S') if current_status is None: query = "INSERT INTO datafilestatus (status, createdate, lastbackuptime, lastbackupmethod, datasetidentifier, storageidentifier) VALUES (%s, %s, %s, %s, %s, %s);" else: query = "UPDATE datafilestatus SET status=%s, createdate=%s, lastbackuptime=%s, lastbackupmethod=%s WHERE datasetidentifier=%s AND storageidentifier=%s;" dataset_id = dataset_authority + "/" + dataset_identifier backup_method = ConfigSectionMap("Backup")['storagetype'] cursor.execute(query, (status, createdate_str, nowdate_str, backup_method, dataset_id, storage_identifier)) # finalize transaction: backup_db_connection.commit() cursor.close()
def open_ssh_client(): ssh_host = ConfigSectionMap("Backup")['sshhost'] ssh_port = ConfigSectionMap("Backup")['sshport'] ssh_username = ConfigSectionMap("Backup")['sshusername'] print "SSH Host: %s" % (ssh_host) print "SSH Port: %s" % (ssh_port) print "SSH Username: %s" % (ssh_username) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_client.connect(hostname=ssh_host, username=ssh_username) print "Connected!" return ssh_client
def open_storage_object_filesystem(dataset_authority, dataset_identifier, object_location, is_tabular_data): filesystem_directory = ConfigSectionMap( "Repository")['filesystemdirectory'] if (is_tabular_data is not None): object_location += ".orig" file_path = filesystem_directory + "/" + dataset_authority + "/" + dataset_identifier + "/" + object_location byte_stream = io.open(file_path, "rb") return byte_stream
def backup_file_swift (file_input, dataset_authority, dataset_identifier, storage_identifier): auth_url = ConfigSectionMap("Backup")['swiftauthurl'] auth_version = ConfigSectionMap("Backup")['swiftauthversion'] user = ConfigSectionMap("Backup")['swiftuser'] tenant = ConfigSectionMap("Backup")['swifttenant'] key = ConfigSectionMap("Backup")['swiftkey'] conn = swiftclient.Connection( authurl=auth_url, user=user, key=key, tenant_name=tenant, auth_version=auth_version ) container_name = dataset_authority + ":" + dataset_identifier conn.put(container_name) conn.put_object(container_name, storage_identifier, file_input)
def backup_file(file_input, dataset_authority, dataset_identifier, storage_identifier, checksum_type, checksum_value, file_size): storage_type = ConfigSectionMap("Backup")['storagetype'] if storage_type == 'swift': #backup_file_swift(file_input, dataset_authority, dataset_identifier, storage_identifier, checksum_type, checksum_value, file_size) raise NotImplementedError('no backup_swift yet') elif storage_type == 'ssh': backup_file_ssh(file_input, dataset_authority, dataset_identifier, storage_identifier, checksum_type, checksum_value, file_size) else: raise ValueError( "only ssh/sftp and swift are supported as backup storage media")
def send_notification(text): try: notification_address = ConfigSectionMap("Notifications")['email'] except: notification_address = None if (notification_address is None): raise ValueError('Notification email address is not configured') nowdate_str = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M') subject_str = ('Dataverse datafile backup report [%s]' % nowdate_str) p = Popen(['mail','-s',subject_str,notification_address], stdout=PIPE, stdin=PIPE, stderr=PIPE) stdout_data = p.communicate(input=text)[0]
def create_database_connection(database='database'): Host = ConfigSectionMap("Database")['host'] Port = ConfigSectionMap("Database")['port'] Database = ConfigSectionMap("Database")[database] Username = ConfigSectionMap("Database")['username'] Password = ConfigSectionMap("Database")['password'] #print "Database Host: %s" % (Host) #print "Database Port: %s" % (Port) #print "Database Name: %s" % (Database) #print "Username: %s" % (Username) #print "Password: %s" % (Password) #Define our connection string conn_string = "host='" + Host + "' dbname='" + Database + "' user='******' password='******'" #print "Connecting to database\n->%s" % (conn_string) # get a connection, if a connect cannot be made an exception will be raised here conn = psycopg2.connect(conn_string) #print "Connected!\n" return conn
def transfer_file(local_flo, dataset_authority, dataset_identifier, storage_identifier, byte_size): sftp_client = my_ssh_client.open_sftp() remote_dir = dataset_authority + "/" + dataset_identifier subdirs = remote_dir.split("/") cdir = ConfigSectionMap("Backup")['backupdirectory'] + "/" for subdir in subdirs: try: cdir = cdir + subdir + "/" sftpattr = sftp_client.stat(cdir) except IOError: #print "directory "+cdir+" does not exist (creating)" sftp_client.mkdir(cdir) #else: # print "directory "+cdir+" already exists" m = re.search('^([a-z0-9]*)://(.*)$', storage_identifier) if m is not None: storageTag = m.group(1) storage_identifier = re.sub('^.*:', '', storage_identifier) remote_file = cdir + storage_identifier if (type(local_flo) is str): sftp_client.put(local_flo, remote_file) else: # assume it's a stream: # sftp_client.putfo() is convenient, but appears to be unavailable in older # versions of paramiko; so we'll be using .read() and .write() instead: #sftp_client.putfo(local_flo,remote_file,byte_size) sftp_stream = sftp_client.open(remote_file, "wb") while True: buffer = local_flo.read(32 * 1024) if len(buffer) == 0: break sftp_stream.write(buffer) sftp_stream.close() sftp_client.close() print "File transfered." return remote_file
from config import ConfigSectionMap led_pin = int(ConfigSectionMap("dust")['led_pin']) try: import RPi.GPIO as GPIO GPIO.setmode(GPIO.BCM) led = True GPIO.setup(led_pin, GPIO.OUT) except ImportError: led = False def on(): if led is True: GPIO.output(led_pin, GPIO.HIGH) def off(): if led is True: GPIO.output(led_pin, GPIO.LOW)
from config import ConfigSectionMap input_pin = int(ConfigSectionMap("gpio")['input_pin']) rs = int(ConfigSectionMap("lcd")['rs']) en = int(ConfigSectionMap("lcd")['en']) d4 = int(ConfigSectionMap("lcd")['d4']) d5 = int(ConfigSectionMap("lcd")['d5']) d6 = int(ConfigSectionMap("lcd")['d6']) d7 = int(ConfigSectionMap("lcd")['d7']) try: import Adafruit_CharLCD as LCD import RPi.GPIO as GPIO except ImportError: lcd = None else: lcd = LCD.Adafruit_CharLCD(rs, en, d4, d5, d6, d7, 16, 2) GPIO.setmode(GPIO.BCM) GPIO.setup(input_pin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN) def GPIO_input(): if lcd is not None: return GPIO.input(input_pin) else: return False def message(msg): if lcd is not None: lcd.message(msg)
import datetime import time from config import ConfigSectionMap from run import run, reset, button_pushed, lcd_ready from utils.timer import int_time, timer_list interval = int(ConfigSectionMap("run")['interval']) duration = int(ConfigSectionMap("run")['duration']) schedule_time = ConfigSectionMap("schedule")['time'] timers = [] start_time = int_time() elapsed_time = 0 schedule_toggle = False def check_schedule(time_str): global schedule_toggle if schedule_toggle is True: return False scheduled_time = time.strptime(time_str, "%H:%M") current_time = datetime.datetime.now() if scheduled_time.tm_hour == current_time.hour and scheduled_time.tm_min == current_time.minute: return True else: return False
# -*- coding: utf-8 -*- from __future__ import unicode_literals import re from config import ConfigSectionMap route_id = ConfigSectionMap("bus")['route_id'] def parse_time_string(time_string): """ parse time string into arrival object :param time_string: "12분33초후[2번째 전]" :return: list (minutes, seconds, stops) """ pattern = re.compile( '(?:(?P<min>\d+)분|)(?:(?P<sec>\d+)초|)후\[(?P<stops>\d+)번째 전\]') match = pattern.search(time_string) s = None if match is not None: m = match.group('min') s = match.group('sec') stops = match.group('stops') # digit if m is None: # "33초후[2번째 전]" m = 0 if s is None: # "12분후[10번째 전] s = 0 s = (int(m), int(s), int(stops))
from bus.bus_query import create_bus_qs from utils.url import make_url import grequests from config import ConfigSectionMap arrival_api_url = 'http://ws.bus.go.kr/api/rest/stationinfo/getStationByUid' key = ConfigSectionMap("api_key")['bus'] station_id = ConfigSectionMap("bus")['station_id'] def bus_grequests(url, key, station_id): bus_qs = create_bus_qs(key, station_id) return grequests.get(make_url(url, bus_qs)) def bus_request(): return bus_grequests(arrival_api_url, key, station_id)
from bs4 import BeautifulSoup import uuid import json import os import boto3 from boto3.s3.transfer import S3Transfer import requests import wget import ssl from functools import wraps from config import ConfigSectionMap from houzzCrawler import Data cfg = ConfigSectionMap('aws') def sslwrap(func): @wraps(func) def bar(*args, **kw): kw['ssl_version'] = ssl.PROTOCOL_TLSv1 return func(*args, **kw) return bar ssl.wrap_socket = sslwrap(ssl.wrap_socket) client = boto3.client('s3', aws_access_key_id=cfg['aws_access_key_id'], aws_secret_access_key=cfg['aws_secret_access_key'])
import datetime from xml.etree import ElementTree import grequests from bus.bus_request import bus_request from bus.bus_view import arrivals_from_xml, bus_arrival_view from dust.dust_request import dust_request from dust.dust_view import dust_from_xml, dust_view from gpio import lcd from gpio import led from weather.weather_request import live_weather_request, forecast_weather_request from weather.weather_view import weather_live_from_json, weather_forecast_from_json, weather_view from config import ConfigSectionMap pm10_threshold = int(ConfigSectionMap("dust")['pm10_threshold']) pm25_threshold = int(ConfigSectionMap("dust")['pm25_threshold']) def run(): lcd.clear() lcd.message("Loading...") weather_live_req = live_weather_request() weather_forecast_req = forecast_weather_request() bus_arrival_req = bus_request() dust_req = dust_request() grequests.map( (weather_live_req, weather_forecast_req, bus_arrival_req, dust_req) )
from dust.dust_query import create_dust_qs from utils.url import make_url import grequests from config import ConfigSectionMap dust_api_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getMsrstnAcctoRltmMesureDnsty' key = ConfigSectionMap("api_key")['dust'] station_name = ConfigSectionMap("dust")['station_name'] def dust_grequests(url, key, station_id): dust_qs = create_dust_qs(key, station_id) return grequests.get(make_url(url, dust_qs)) def dust_request(): return dust_grequests(dust_api_url, key, station_name)
from flask import Flask, flash, request, jsonify, render_template, redirect import json import os import sys import urllib.request from werkzeug.utils import secure_filename from flask_dropzone import Dropzone #Modulos Internos from modules.lotes_site.lotesite import getlotesite from modules.detail_lote_site.detaillotesite import getdetaillotesite from app import UPLOAD_FOLDER from config import ConfigSectionMap #Configuraciones Iiciales extension = ConfigSectionMap('Extension') separator = ConfigSectionMap('Separator') fileencode = ConfigSectionMap('FileEncode') EXTENSION = extension['extensionallow'] EXTENSION = (json.loads(EXTENSION.replace("'", '"'))) SEPARATOR = separator['type'] SEPARATOR = (json.loads(SEPARATOR.replace("'", '"'))) FILEENCODE = fileencode['typefile'] FILEENCODE = (json.loads(FILEENCODE.replace("'", '"'))) app = Flask(__name__) dropzone = Dropzone(app) UPLOAD_FOLDER = '/home/quintanada/Pictures/upload/' app.secret_key = "secret key" app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
from utils.url import make_url from config import ConfigSectionMap from weather.weather_query import create_weather_live_qs, create_weather_forecast_qs import grequests import datetime forecast_api_url = 'http://newsky2.kma.go.kr/service/SecndSrtpdFrcstInfoService2/ForecastSpaceData' live_api_url = 'http://newsky2.kma.go.kr/service/SecndSrtpdFrcstInfoService2/ForecastGrib' nx = int(ConfigSectionMap("weather")['nx']) ny = int(ConfigSectionMap("weather")['ny']) live_basetime_delta = int(ConfigSectionMap("weather_basetime_delta")['live']) forecast_basetime_delta = int(ConfigSectionMap("weather_basetime_delta")['forecast']) key = ConfigSectionMap("api_key")['weather'] def live_weather_request(): now = datetime.datetime.now() qs = create_weather_live_qs(key, nx, ny, now, live_basetime_delta) return grequests.get(make_url(live_api_url, qs)) def forecast_weather_request(): now = datetime.datetime.now() qs = create_weather_forecast_qs(key, nx, ny, now, forecast_basetime_delta) return grequests.get(make_url(forecast_api_url, qs))
import requests from bs4 import BeautifulSoup from requests.packages.urllib3.util.retry import Retry from requests.adapters import HTTPAdapter import ssl from functools import wraps from config import ConfigSectionMap cfg = ConfigSectionMap('setup') START = cfg['start'] END = cfg['end'] session = requests.Session() retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) session.proxies = proxies session.headers = headers session.mount('https://', HTTPAdapter(max_retries=retries)) def sslwrap(func): @wraps(func) def bar(*args, **kw): kw['ssl_version'] = ssl.PROTOCOL_TLSv1 return func(*args, **kw) return bar ssl.wrap_socket = sslwrap(ssl.wrap_socket)
import requests from config import ConfigSectionMap from bs4 import BeautifulSoup import boto3 from boto3.s3.transfer import S3Transfer import wget import os import json import uuid cfg_setup = ConfigSectionMap('setup') class Profile: def __init__(self, soup): self.soup = soup self.name = self.extract_name(self.soup) self.user_name = self.extract_username(self.soup) self.profile_pic = self.extract_profile_pic(self.soup) self.description = self.extract_description(self.soup) self.services_provided = self.extract_services_provided(self.soup) self.areas_serviced = self.extract_areas_serviced(self.soup) self.social_details = self.extract_social_details(self.soup) self.profession = self.extract_profession(self.soup) self.contact_details = self.extract_contact_details(self.soup) self.followers = self.extract_followers(self.soup) def __repr__(self): return repr( json.dumps({ "name": self.name,