def get_campaigns_by_bloodtype():
session = db.Session()
user_id = request.args.get('user_id', 0)
# filter by user Blood Type
user = session.query(db.User).filter_by(user_id=user_id).first()
if not user:
session.close()
return ApiResponse({
'status':
'error',
'message':
'No user with id {0} found'.format(user_id)
})
campaigns_blood = session.query(
db.CampaignBlood).filter_by(blood_type=user.blood_type).all()
campaigns = [{
'name': c.campaign.name,
'hospital': {
'name': c.campaign.hospital.name,
'latitude': c.campaign.hospital.latitude,
'longitude': c.campaign.hospital.longitude,
},
'message': c.campaign.message,
'start_date': to_timestamp(c.campaign.start_date),
'end_date': to_timestamp(c.campaign.end_date)
} for c in campaigns_blood]
session.close()
# return data
return ApiResponse({"campaigns": campaigns})
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this Tag.
"""
return {
"name":
self.name,
"time_type":
self.time_type,
"object_type":
self.object_type,
"synchronization":
self.synchronization,
"end_of_validity":
self.end_of_validity,
"description":
self.description,
"last_validated_time":
self.last_validated_time,
"insertion_time":
to_timestamp(self.insertion_time)
if convert_timestamps else self.insertion_time,
"modification_time":
to_timestamp(self.modification_time)
if convert_timestamps else self.modification_time,
"record":
self.record,
"label":
self.label
}
def to_array(self):
return [
self.queue, self.tag, self.record, self.label,
status_full_name(self.status),
to_timestamp(self.time_submitted),
to_timestamp(self.last_edited)
]
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of Global Tag object.
"""
json_gt = {
'name':
self.name,
'validity':
self.validity,
'description':
self.description,
'release':
self.release,
'insertion_time':
to_timestamp(self.insertion_time)
if convert_timestamps else self.insertion_time,
'snapshot_time':
to_timestamp(self.snapshot_time)
if convert_timestamps else self.snapshot_time,
'scenario':
self.scenario,
'workflow':
self.workflow,
'type':
self.type
}
return json_gt
def get_campaigns_by_bloodtype():
session = db.Session()
user_id = request.args.get('user_id', 0)
# filter by user Blood Type
user = session.query(db.User).filter_by(user_id=user_id).first()
if not user:
session.close()
return ApiResponse({
'status': 'error',
'message': 'No user with id {0} found'.format(user_id)
})
campaigns_blood = session.query(db.CampaignBlood).filter_by(blood_type=user.blood_type).all()
campaigns = [
{
'name': c.campaign.name,
'hospital': {
'name': c.campaign.hospital.name,
'latitude': c.campaign.hospital.latitude,
'longitude': c.campaign.hospital.longitude,
},
'message': c.campaign.message,
'start_date': to_timestamp(c.campaign.start_date),
'end_date': to_timestamp(c.campaign.end_date)
} for c in campaigns_blood]
session.close()
# return data
return ApiResponse({
"campaigns": campaigns
})
def create_campaign():
session = db.Session()
data = json.loads(request.data)
hospital_id = request.args.get('hospital_id', 0)
# hospital = session.query(db.Hospital).filter_by(_id=hospital_id).first()
hospital = session.query(db.Hospital).first()
name = data['name']
message = data['message']
bloodtypes = data['bloodtypes']
start_date = datetime.datetime.now()
end_date = datetime.datetime.now() + datetime.timedelta(days=10)
campaign = db.Campaign(hospital._id, name, message, start_date, end_date)
session.add(campaign)
session.commit()
for bloodtype in bloodtypes:
campaign_blood = db.CampaignBlood(campaign._id, bloodtype)
session.add(campaign_blood)
session.commit()
gcmClient = GCMClient(api_key=os.environ.get('GCM_API_KEY'))
alert = {
'subject': 'Fushate e re',
'message': campaign.hospital.name,
'data': {
'id': campaign._id,
'name': name,
'hospital': {
'name': campaign.hospital.name,
'latitude': campaign.hospital.latitude,
'longitude': campaign.hospital.longitude,
},
'message': message,
'start_date': to_timestamp(start_date),
'end_date': to_timestamp(end_date)
}
}
interested_users = session.query(db.User).filter(db.User.blood_type.in_(bloodtypes))
gcm_id_list = [user.gcm_id for user in interested_users]
session.close()
response = gcmClient.send(gcm_id_list, alert, time_to_live=3600)
if response:
return ApiResponse({
'status': 'ok'
})
else:
return ApiResponse({
'status': 'some error occurred'
})
def output_date(self, date, start, dt_format=None):
"""
Output lines with the date.
Output starts from the block at "start" position and
ends when the date is no longer met.
Next blocks reads as needed.
:param date: string with some date
:type date: str
:param start: a start position of a block
:type start: int
:param dt_format: format of the date
:type dt_format: str
"""
if dt_format:
self._set_dt_format(dt_format)
stamp = to_timestamp(date, self.dt_format)
block = _Range(start, self._get_end_of_block(start))
rest = self._print_stamp_from_block(block, stamp)
while rest:
block_start = block.end
block_end = self._get_end_of_block(block_start)
rest = self._print_stamp_from_block(
_Range(block_start, block_end), stamp, rest)
def user_past_donations(user_id=None):
session = db.Session()
if user_id is None:
user_id = request.args.get('user_id', 0)
user = session.query(db.User).filter_by(user_id=user_id).first()
if not user:
session.close()
return ApiResponse({
'status': 'error',
'message': 'No user with id {0} found'.format(id)
})
donations = session.query(db.UserHistory).filter_by(user_id=user.user_id).all()
result = {
'user': user.user_id,
'history': [{
'date': to_timestamp(d.donation_date),
'amount': d.amount,
'hospital': d.hospital.name
} for d in donations]
}
session.close()
return ApiResponse({
'history': result
})
def user_past_donations(user_id=None):
session = db.Session()
if user_id is None:
user_id = request.args.get('user_id', 0)
user = session.query(db.User).filter_by(user_id=user_id).first()
if not user:
session.close()
return ApiResponse({
'status': 'error',
'message': 'No user with id {0} found'.format(id)
})
donations = session.query(
db.UserHistory).filter_by(user_id=user.user_id).all()
result = {
'user':
user.user_id,
'history': [{
'date': to_timestamp(d.donation_date),
'amount': d.amount,
'hospital': d.hospital.name
} for d in donations]
}
session.close()
return ApiResponse({'history': result})
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of Global Tag object.
"""
json_gt = {
'name': self.name,
'validity': self.validity,
'description': self.description,
'release': self.release,
'insertion_time': to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time,
'snapshot_time': to_timestamp(self.snapshot_time) if convert_timestamps else self.snapshot_time,
'scenario': self.scenario,
'workflow': self.workflow,
'type': self.type
}
return json_gt
def _adjust_timecode(episode, timestamp):
'''
Offset a timecode by the total number of offset frames
'''
frame = timestamp_to_seconds(timestamp)
offsets = episode.offsets
if episode.is_pioneer:
offsets = episode.pioneer_offsets
series = episode.series
total_offset = 0
# calculate offset from frame data
if isinstance(offsets, list):
# for list-types (movies, not episodes), start with 0 offset
for o in offsets:
if frame > frame_to_seconds(o['frame']):
total_offset += frame_to_seconds(o['offset'])
else:
# episodes are map-based, with a key for each chapter
# orange bricks have a delay on the OP subs
if (series == 'DBZ' and not episode.is_r1dbox and
frame < frame_to_seconds(offsets['prologue']["frame"])):
total_offset += _op_subtitle_delay(episode)
for key in offsets.keys():
# also account for ED subs being +0.333 s early
if frame > frame_to_seconds(offsets[key]["frame"]):
total_offset += frame_to_seconds(
offsets[key]["offset"])
# apply offset to subtitle timing
frame -= total_offset
return to_timestamp(frame)
def send_blob(self, payload, upload_session_id):
"""
Send the BLOB of a payload over HTTP.
The BLOB is put in the request body, so no additional processing has to be done on the server side, apart from decoding from base64.
"""
# encode the BLOB data of the Payload to make sure we don't send a character that will influence the HTTPs request
blob_data = base64.b64encode(payload["data"])
url_data = {"database" : self.data_to_send["destinationDatabase"], "upload_session_id" : upload_session_id}
# construct the data to send in the body and header of the HTTPs request
for key in payload.keys():
# skip blob
if key != "data":
if key == "insertion_time":
url_data[key] = to_timestamp(payload[key])
else:
url_data[key] = payload[key]
request = url_query(url=self._SERVICE_URL + "store_payload/", url_data=url_data, body=blob_data)
# send the request and return the response
# Note - the url_query module will handle retries, and will throw a NoMoreRetriesException if it runs out
try:
request_response = request.send()
return request_response
except Exception as e:
# make sure we don't try again - if a NoMoreRetriesException has been thrown, retries have run out
if isinstance(e, errors.NoMoreRetriesException):
self._outputter.write("\t\t\tPayload with hash '%s' was not uploaded because the maximum number of retries was exceeded." % payload["hash"])
self._outputter.write("Payload with hash '%s' was not uploaded because the maximum number of retries was exceeded." % payload["hash"])
return json.dumps({"error" : str(e), "traceback" : traceback.format_exc()})
def add_satz(self, fullname, result, date):
s = self.get_schuetze_by_fullname(fullname)
entry = JSONSatz(schuetze_uuid=s.uuid,
result=result,
date=utils.to_timestamp(date))
self.data.append(entry)
self._generic_add_data(entry, self.settings.data_file)
return entry
def post():
"""
Creates new db instance of Employee. position_id, department_id, name and surname args are needed.
Other info (email, phone, birth_date, start_work_date and is_department_leader) is optional.
If start_work_date is not set -> will be set to now.
If is_department_leader is not set -> will be set to False.
:return: success/error json
"""
vacancy_id = request.form.get('vacancy_id')
name = request.form.get('name')
surname = request.form.get('surname')
email = request.form.get('email')
phone = request.form.get('phone')
birth_date = to_timestamp(request.form.get('birth_date'))
start_work_date = to_timestamp(request.form.get('start_work_date'))
is_department_leader = request.form.get('is_department_leader')
if not all((vacancy_id, name, surname)):
return {'error': 'vacancy_id, name, surname required'}, 400
else:
try:
vacancy = Vacancy.query.filter_by(id=vacancy_id).first()
employee = Employee(position_id=vacancy.position.id,
department_id=vacancy.department.id,
name=name,
surname=surname,
email=email,
phone=phone,
birth_date=birth_date,
start_work_date=start_work_date,
is_department_leader=is_department_leader)
db.session.add(employee)
db.session.commit()
return {
'success': 'employee created',
'employee': {
'id': employee.id,
'name': employee.name,
'surname': employee.surname,
'position': employee.position.name
}
}
except Exception as e:
return {
'error': 'failed to create employee: {}'.format(e)
}, 400
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this Tag.
"""
return {
"name" : self.name,
"time_type" : self.time_type,
"object_type" : self.object_type,
"synchronization" : self.synchronization,
"end_of_validity" : self.end_of_validity,
"description" : self.description,
"last_validated_time" : self.last_validated_time,
"insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time,
"modification_time" : to_timestamp(self.modification_time) if convert_timestamps else self.modification_time,
"record" : self.record,
"label" : self.label
}
def add_satz(self, fullname, result, date):
s = self.get_schuetze_by_fullname(fullname)
entry = JSONSatz(
schuetze_uuid=s.uuid,
result=result,
date=utils.to_timestamp(date))
self.data.append(entry)
self._generic_add_data(entry, self.settings.data_file)
return entry
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this IOV.
"""
return {
"tag_name" : self.tag_name,
"since" : self.since,
"payload_hash" : self.payload_hash,
"insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
}
def __init__(self, dictionary={}, convert_timestamps=True):
# assign each entry in a kwargs
for key in dictionary:
try:
if convert_timestamps:
self.__dict__[key] = to_timestamp(dictionary[key])
else:
self.__dict__[key] = dictionary[key]
except KeyError as k:
continue
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this Payload's metadata (not the actual Payload).
"""
return {
"hash" : self.hash,
"object_type" : self.object_type,
"version" : self.version,
"insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
}
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this Payload's metadata (not the actual Payload).
"""
return {
"hash" : self.hash,
"object_type" : self.object_type,
"version" : self.version,
"insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
}
def __init__(self, dictionary={}, convert_timestamps=True):
# assign each entry in a kwargs
for key in dictionary:
try:
if convert_timestamps:
self.__dict__[key] = to_timestamp(dictionary[key])
else:
self.__dict__[key] = dictionary[key]
except KeyError as k:
continue
def as_dicts(self, convert_timestamps=False):
"""
Returns dictionary form of this IOV.
"""
return {
"tag_name" : self.tag_name,
"since" : self.since,
"payload_hash" : self.payload_hash,
"insertion_time" : to_timestamp(self.insertion_time) if convert_timestamps else self.insertion_time
}
def save_progress(self, last_block, last_timestamp, last_hash):
self.conn.execute("DELETE FROM last_block")
self.conn.execute(
"INSERT INTO last_block VALUES (?, ?, ?) ",
(last_block, to_timestamp(last_timestamp), last_hash))
self.conn.execute("DELETE FROM schema_version")
self.conn.execute("INSERT INTO schema_version VALUES (?) ",
(Logger.SCHEMA_VERSION, ))
self.conn.commit()
def safe_handler(*args, **kwargs):
session_token = request.args.get('session_token', '')
user_id = request.args.get('user_id', 0)
user = session.query(User).filter_by(user_id=user_id).first()
sleep_amount = random.random()/5
if user and utils.str_equal(user.session_token, session_token) and \
utils.to_timestamp(user.session_token_expires_at) > time.time():
return handler(*args, **kwargs)
else:
time.sleep(sleep_amount)
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
def all_campaigns():
session = db.Session()
hospital_id = request.args.get('hospital_id', 0)
#campaigns = session.query(db.Campaign).filter_by(hospital_id=hospital_id).all()
campaigns = session.query(db.Campaign).all()
bloodtypes = session.query()
response = ApiResponse({
'campaigns': [
{
'id': c._id,
'name': c.name,
'message': c.message,
'start_date': to_timestamp(c.start_date),
'end_date': to_timestamp(c.end_date),
'active': c.active,
'bloodtypes': [r.blood_type for r in c.requirement]
} for c in campaigns]
})
session.close()
return response
def post():
"""
Opens vacancy in current department. department_id and position_id are needed.
date_opened is optional (defaults to now)
:return: success/error json
"""
department_id = request.form.get('department_id')
position_id = request.form.get('position_id')
date_opened = request.form.get('date_opened')
if not position_id or not department_id:
return {'error': 'position_id and department_id required'}, 400
else:
try:
department = Department.query.filter_by(
id=department_id).first()
position = Position.query.filter_by(id=position_id).first()
except DataError:
return {
'error': 'department_id and position_id must be ints'
}, 400
if not department or not position:
return {'error': 'department or position not found'}, 404
else:
if date_opened:
try:
date_opened = to_timestamp(date_opened)
vacancy = Vacancy(department_id=department_id,
position_id=position_id,
date_opened=date_opened)
except ValueError:
return {
'error': 'date_opened must be in format mm/dd/yyyy'
}, 400
else:
vacancy = Vacancy(department_id=department_id,
position_id=position_id)
db.session.add(vacancy)
db.session.commit()
return {
'success': 'vacancy created',
'vacancy': {
'id': vacancy.id,
'department_id': vacancy.department_id,
'date_opened': vacancy.str_date_opened,
'position': {
'id': vacancy.position.id,
'name': vacancy.position.name
}
}
}
def safe_handler(*args, **kwargs):
session = db.Session()
session_token = request.args.get('session_token', '')
user_id = request.args.get('user_id', 0)
user = session.query(db.User).filter_by(user_id=user_id).first()
if user and utils.str_equal(user.session_token, session_token) and \
utils.to_timestamp(user.session_token_expires_at) > time.time():
response = handler(*args, **kwargs)
else:
response = ApiResponse(config.ACCESS_DENIED_MSG, status='403')
session.close()
return response
def patch():
"""
Updates vacancy data.
:return: success/error json
"""
vacancy_id = request.form.get('vacancy_id')
new_department_id = request.form.get('new_department_id')
new_position_id = request.form.get('new_position_id')
new_date_opened = request.form.get('new_date_opened')
if not vacancy_id or not all(
(new_department_id, new_position_id, new_date_opened)):
return {
'error':
'vacancy_id and at least one of new_department_id, new_position_id, new_date_opened needed'
}, 400
else:
try:
vacancy = Vacancy.query.filter_by(id=vacancy_id).first()
if not vacancy:
return {'error': 'vacancy not found'}, 404
if new_department_id:
vacancy.department_id = new_department_id
if new_position_id:
vacancy.position_id = new_position_id
if new_date_opened:
try:
vacancy.date_opened = to_timestamp(new_date_opened)
except ValueError:
return {
'error': 'date_opened must be in format mm/dd/yyyy'
}, 400
db.session.add(vacancy)
db.session.commit()
return {
'success': 'vacancy updated',
'vacancy': {
'id': vacancy.id,
'date_opened': vacancy.date_opened,
'department': {
'id': vacancy.department_id,
'name': vacancy.department.name
},
'position': {
'id': vacancy.position_id,
'name': vacancy.position.name
}
}
}
except (DataError, IntegrityError):
return {'error': 'bad department_id, position_id, vacancy_id'}
def login():
data = json.loads(request.data)
user_id = data['user_id']
gcmID = data['gcmID']
fb_token = data['fb_token']
payload = {'access_token': fb_token, 'fields': 'id'}
fb_response = requests.get(config.FB_ENDPOINT, params=payload).json()
if 'error' in fb_response:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
elif user_id != fb_response['id']:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
# Facebook login was successful
user = session.query(User).filter_by(user_id=user_id).first()
gcm_id = request.args.get('gcm_id', '')
blood_type = request.args.get('blood_type', '')
if user:
user.fb_token = fb_token
token, expires_at = User.generate_session_token()
user.session_token = token
user.session_token_expires_at = expires_at
if gcm_id:
user.gcm_id = gcm_id
if blood_type:
user.blood_type = blood_type
session.commit()
else:
user = User(user_id,
fb_token=fb_token,
gcm_id=gcm_id,
blood_type=blood_type)
session.add(user)
session.commit()
if user:
return ApiResponse({
'status':
'OK',
'session_token':
user.session_token,
'expires_at':
to_timestamp(user.session_token_expires_at)
})
else:
return ApiResponse({
'status': 'Failed',
'message': "Couldn't create new user"
})
def dashboard():
ops = db_fieldbook.get_all_opportunities()
for op in ops:
if op["status"] == "Accepted":
op["class"] = "success"
elif op["status"] == "Offered":
op["class"] = "info"
elif op["status"] == "Expired":
op["class"] = "active"
elif op["status"] == "Attended":
op["class"] = "active"
elif op["status"] == "Not Attended":
op["class"] = "active"
op["remaining_mins"] = int(int(op["expiry_time"] - utils.to_timestamp(datetime.datetime.utcnow())) / 60)
return render_template('dashboard.html', ops=ops, dash_refresh_timeout=config.dash_refresh_timeout)
def login():
data = json.loads(request.data)
user_id = data['user_id']
gcmID = data['gcmID']
fb_token = data['fb_token']
payload= {
'access_token': fb_token,
'fields': 'id'
}
fb_response = requests.get(config.FB_ENDPOINT, params=payload).json()
if 'error' in fb_response:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
elif user_id != fb_response['id']:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
# Facebook login was successful
user = session.query(User).filter_by(user_id=user_id).first()
gcm_id = request.args.get('gcm_id', '')
blood_type = request.args.get('blood_type', '')
if user:
user.fb_token = fb_token
token, expires_at = User.generate_session_token()
user.session_token = token
user.session_token_expires_at = expires_at
if gcm_id:
user.gcm_id = gcm_id
if blood_type:
user.blood_type = blood_type
session.commit()
else:
user = User(user_id, fb_token=fb_token, gcm_id=gcm_id,
blood_type=blood_type)
session.add(user)
session.commit()
if user:
return ApiResponse({
'status': 'OK',
'session_token': user.session_token,
'expires_at': to_timestamp(user.session_token_expires_at)
})
else:
return ApiResponse({
'status': 'Failed',
'message': "Couldn't create new user"
})
def login():
session = db.Session()
data = json.loads(request.data)
user_id = data['user_id']
gcm_id = data['gcm_id']
fb_token = data['fb_token']
payload = {'access_token': fb_token, 'fields': ['id', 'name']}
fb_response = requests.get(config.FB_ENDPOINT, params=payload).json()
if 'error' in fb_response:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
elif user_id != fb_response['id']:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
# Facebook login was successful
user = session.query(db.User).filter_by(user_id=user_id).first()
if user:
user.fb_token = fb_token
token, expires_at = db.User.generate_session_token()
user.session_token = token
user.session_token_expires_at = expires_at
if gcm_id:
user.gcm_id = gcm_id
else:
name = fb_response['name'].split()
user = db.User(user_id,
name[0],
name[-1],
fb_token=fb_token,
gcm_id=gcm_id)
#blood_type=blood_type)
session.add(user)
session.commit()
response = ApiResponse(
{
'status': 'OK',
'session_token': user.session_token,
'expires_at': to_timestamp(user.session_token_expires_at)
} if user else {
'status': 'Failed',
'message': "Couldn't create new user"
})
session.close()
return response
def login():
session = db.Session()
data = json.loads(request.data)
user_id = data['user_id']
gcm_id = data['gcm_id']
fb_token = data['fb_token']
payload = {
'access_token': fb_token,
'fields': ['id', 'name']
}
fb_response = requests.get(config.FB_ENDPOINT, params=payload).json()
if 'error' in fb_response:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
elif user_id != fb_response['id']:
return ApiResponse(config.ACCESS_DENIED_MSG, status='403')
# Facebook login was successful
user = session.query(db.User).filter_by(user_id=user_id).first()
if user:
user.fb_token = fb_token
token, expires_at = db.User.generate_session_token()
user.session_token = token
user.session_token_expires_at = expires_at
if gcm_id:
user.gcm_id = gcm_id
else:
name = fb_response['name'].split()
user = db.User(user_id, name[0], name[-1], fb_token=fb_token, gcm_id=gcm_id)
#blood_type=blood_type)
session.add(user)
session.commit()
response = ApiResponse({
'status': 'OK',
'session_token': user.session_token,
'expires_at': to_timestamp(user.session_token_expires_at)
} if user else {
'status': 'Failed',
'message': "Couldn't create new user"
})
session.close()
return response
def send_blob(self, payload, upload_session_id):
"""
Send the BLOB of a payload over HTTP.
The BLOB is put in the request body, so no additional processing has to be done on the server side, apart from decoding from base64.
"""
# encode the BLOB data of the Payload to make sure we don't send a character that will influence the HTTPs request
blob_data = base64.b64encode(payload["data"])
url_data = {
"database": self.data_to_send["destinationDatabase"],
"upload_session_id": upload_session_id
}
# construct the data to send in the body and header of the HTTPs request
for key in payload.keys():
# skip blob
if key != "data":
if key == "insertion_time":
url_data[key] = to_timestamp(payload[key])
else:
url_data[key] = payload[key]
request = url_query(url=self._SERVICE_URL + "store_payload/",
url_data=url_data,
body=blob_data)
# send the request and return the response
# Note - the url_query module will handle retries, and will throw a NoMoreRetriesException if it runs out
try:
request_response = request.send()
return request_response
except Exception as e:
# make sure we don't try again - if a NoMoreRetriesException has been thrown, retries have run out
if type(e) == errors.NoMoreRetriesException:
self._outputter.write(
"\t\t\tPayload with hash '%s' was not uploaded because the maximum number of retries was exceeded."
% payload["hash"])
self._outputter.write(
"Payload with hash '%s' was not uploaded because the maximum number of retries was exceeded."
% payload["hash"])
return json.dumps({
"error": str(e),
"traceback": traceback.format_exc()
})
def seek(self, date, dt_format=None):
"""
Search the start position of date in a bzipped log file.
:param date: string with some date
:type date: str
:return: position of day
"""
if dt_format:
self._set_dt_format(dt_format)
stamp = to_timestamp(date, self.dt_format)
block = self._get_block_with_date(stamp)
if not block:
return
if block.start == block.end:
block.end = self._get_end_of_block(block.end)
return block
def to_array(self):
return [self.hash, self.object_type, self.version, to_timestamp(self.insertion_time)]
def __init__(self,
metadata_source=None,
debug=False,
verbose=False,
testing=False,
server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
**kwargs):
"""
Upload constructor:
Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded.
Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used.
Note: default value of service_url should be changed for production.
"""
# set private variables
self._debug = debug
self._verbose = verbose
self._testing = testing
# initialise server-side log data as empty string - will be replaced when we get a response back from the server
self._log_data = ""
self._SERVICE_URL = server
self.upload_session_id = None
# set up client-side log file
self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id(
)
self._handle = open(self.upload_log_file_name, "a")
# set up client-side logging object
self._outputter = output(verbose=verbose, log_handle=self._handle)
self._outputter.write("Using server instance at '%s'." %
self._SERVICE_URL)
# expect a CondDBFW data_source object for metadata_source
if metadata_source == None:
# no upload metadat has been given - we cannot continue with the upload
self.exit_upload(
"A source of metadata must be given so CondDBFW knows how to upload conditions."
)
else:
# set up global metadata source variable
self.metadata_source = metadata_source.data()
# check for the destination tag
# this is required whatever type of upload we're performing
if self.metadata_source.get("destinationTags") == None:
self.exit_upload("No destination Tag was given.")
else:
if type(self.metadata_source.get(
"destinationTags")) == dict and self.metadata_source.get(
"destinationTags").keys()[0] == None:
self.exit_upload("No destination Tag was given.")
# make sure a destination database was given
if self.metadata_source.get("destinationDatabase") == None:
self.exit_upload("No destination database was given.")
# get Conditions metadata
if self.metadata_source.get(
"sourceDB") == None and self.metadata_source.get(
"hashToUse") == None:
"""
If we have neither an sqlite file nor the command line data
"""
self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\
+ "\nSee --help for command line argument information.")
elif self.metadata_source.get("sourceDB") != None:
"""
We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source
We now extract the Tag and IOV data from SQLite. It is added to the dictionary for sending over HTTPs later.
"""
# make sure we have an input tag to look for in the source db
self.input_tag = metadata_source.data().get("inputTag")
if self.input_tag == None:
self.exit_upload("No input Tag name was given.")
# set empty dictionary to contain Tag and IOV data from SQLite
result_dictionary = {}
self.sqlite_file_name = self.metadata_source["sourceDB"]
if not (os.path.isfile(self.sqlite_file_name)):
self.exit_upload("SQLite file '%s' given doesn't exist." %
self.sqlite_file_name)
sqlite_con = querying.connect(
"sqlite://%s" % os.path.abspath(self.sqlite_file_name))
self._outputter.write("Getting Tag and IOVs from SQLite database.")
# query for Tag, check for existence, then convert to dictionary
tag = sqlite_con.tag(name=self.input_tag)
if tag == None:
self.exit_upload(
"The source Tag '%s' you gave was not found in the SQLite file."
% self.input_tag)
tag = tag.as_dicts(convert_timestamps=True)
# query for IOVs, check for existence, then convert to dictionaries
iovs = sqlite_con.iov(tag_name=self.input_tag)
if iovs == None:
self.exit_upload(
"No IOVs found in the SQLite file given for Tag '%s'." %
self.input_tag)
iovs = iovs.as_dicts(convert_timestamps=True)
iovs = [iovs] if type(iovs) != list else iovs
"""
Finally, get the list of all Payload hashes of IOVs,
then compute the list of hashes for which there is no Payload for
this is used later to decide if we can continue the upload if the Payload was not found on the server.
"""
iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag)
if iovs_for_hashes.__class__ == data_sources.json_list:
hashes_of_iovs = iovs_for_hashes.get_members(
"payload_hash").data()
else:
hashes_of_iovs = [iovs_for_hashes.payload_hash]
self.hashes_with_no_local_payload = [
payload_hash for payload_hash in hashes_of_iovs
if sqlite_con.payload(hash=payload_hash) == None
]
# close session open on SQLite database file
sqlite_con.close_session()
elif metadata_source.data().get("hashToUse") != None:
"""
Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments).
We now use Tag and IOV data from command line. It is added to the dictionary for sending over HTTPs later.
"""
# set empty dictionary to contain Tag and IOV data from command line
result_dictionary = {}
now = to_timestamp(datetime.now())
# tag dictionary will be taken from the server
# this does not require any authentication
tag = self.get_tag_dictionary()
self.check_response_for_error_key(tag)
iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\
"insertion_time" : now}]
# hashToUse cannot be stored locally (no sqlite file is given), so register it as not found
self.hashes_with_no_local_payload = [
self.metadata_source["hashToUse"]
]
# Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side
# if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0
if tag["time_type"] == "Run":
for (i, iov) in enumerate(iovs):
iovs[i]["since"] = iovs[i]["since"] << 32
result_dictionary = {"inputTagData": tag, "iovs": iovs}
# add command line arguments to dictionary
# remembering that metadata_source is a json_dict object
result_dictionary.update(metadata_source.data())
# store in instance variable
self.data_to_send = result_dictionary
# if the since doesn't exist, take the first since from the list of IOVs
if result_dictionary.get("since") == None:
result_dictionary["since"] = sorted(
iovs, key=lambda iov: iov["since"])[0]["since"]
elif self.data_to_send["inputTagData"]["time_type"] == "Run":
# Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing
self.data_to_send["since"] = self.data_to_send["since"] << 32
"""
TODO - Settle on a single destination tag format.
"""
# look for deprecated metadata entries - give warnings
# Note - we only really support this format
try:
if type(result_dictionary["destinationTags"]) == dict:
self._outputter.write(
"WARNING: Multiple destination tags in a single metadata source is deprecated."
)
except Exception as e:
self._outputter.write("ERROR: %s" % str(e))
def to_array(self):
return [self.since, to_timestamp(self.insertion_time), self.payload_hash]
def add_event(self, date, description):
entry = JSONEvent(
date=utils.to_timestamp(date),
description=description)
self.events.append(entry)
self._generic_add_data(entry, self.settings.event_file)
def insert_fee(self, block_height, block_time, amount):
self.conn.execute("INSERT INTO fees VALUES (?, ?, ?)",
(block_height, to_timestamp(block_time), amount))
def __init__(self, database, bitcoin_rpc, liquid_rpc):
#Initialize Database if not created
self.conn = sqlite3.connect(database)
self.conn.execute(
'''CREATE TABLE if not exists schema_version (version int)''')
self.bitcoin_rpc = bitcoin_rpc
self.liquid_rpc = liquid_rpc
schema_version = self.conn.execute(
"SELECT version FROM schema_version").fetchall()
if len(schema_version) == 0:
self.conn.execute(
'''CREATE TABLE if not exists missing_blocks (datetime int, functionary int)'''
)
self.conn.execute(
'''CREATE TABLE if not exists fees (block int, datetime int, amount int)'''
)
self.conn.execute(
'''CREATE TABLE if not exists outages (end_time int, length int)'''
)
self.conn.execute(
'''CREATE TABLE if not exists pegs (block int, datetime int, amount int, txid string, txindex int)'''
)
self.conn.execute(
'''CREATE TABLE if not exists issuances (block int, datetime int, asset text, amount int NULL, txid string, txindex int, token string NULL, tokenamount int NULL)'''
)
self.conn.execute(
'''CREATE TABLE if not exists last_block (block int, datetime int, block_hash string)'''
)
self.conn.execute(
'''CREATE TABLE if not exists wallet (txid string, txindex int, amount int, block_hash string, block_timestamp string, spent_txid string NULL, spent_index int NULL)'''
)
self.reindex()
else:
if schema_version[0][0] < 2:
self.conn.execute('DROP TABLE issuances')
self.conn.execute('DROP TABLE pegs')
self.conn.execute(
'''CREATE TABLE if not exists pegs (block int, datetime int, amount int, txid string. index int)'''
)
self.conn.execute(
'''CREATE TABLE if not exists issuances (block int, datetime int, asset text, amount int NULL, txid string, txindex int)'''
)
if schema_version[0][0] < 3:
self.conn.execute('DROP TABLE issuances')
self.conn.execute(
'''CREATE TABLE if not exists issuances (block int, datetime int, asset text, amount int NULL, txid string, txindex int, token string NULL, tokenamount int NULL)'''
)
if schema_version[0][0] < 4:
self.conn.execute("DROP TABLE last_block")
self.conn.execute(
'''CREATE TABLE if not exists last_block (block int, datetime int, block_hash string)'''
)
if schema_version[0][0] < 5:
self.conn.execute(
'''CREATE TABLE if not exists wallet (txid string, txindex int, amount int, block_hash string, block_timestamp string, spent_txid string NULL, spent_index int NULL)'''
)
self.reindex()
if schema_version[0][0] == 5 or schema_version[0][0] == 6:
self.reindex()
else:
configuration = self.conn.execute(
"SELECT block, datetime, block_hash FROM last_block"
).fetchall()
should_reindex = False
if len(configuration) == 0:
should_reindex = True
else:
self.last_time = datetime.fromtimestamp(
configuration[0][1])
self.last_block = configuration[0][0]
self.conn.execute(
'''DELETE FROM missing_blocks WHERE datetime >= ? ''',
(to_timestamp(self.last_time), ))
self.conn.execute(
'''DELETE FROM fees WHERE datetime >= ? ''',
(to_timestamp(self.last_time), ))
self.conn.execute(
'''DELETE FROM outages WHERE end_time >= ? ''',
(to_timestamp(self.last_time), ))
self.conn.execute(
'''DELETE FROM pegs WHERE datetime >= ? ''',
(to_timestamp(self.last_time), ))
self.conn.execute(
'''DELETE FROM issuances WHERE datetime >= ? ''',
(to_timestamp(self.last_time), ))
self.block_hash = configuration[0][2]
# Reindex if block hash doesn't check out
should_reindex = \
self.last_block is not None and self.block_hash is not None and \
self.liquid_rpc.getblockhash(self.last_block) != self.block_hash
if should_reindex:
self.reindex()
self.conn.commit()
def insert_peg(self, block_height, block_time, amount, txid, txindex):
self.conn.execute(
"INSERT INTO pegs VALUES (?, ?, ?, ? , ?)",
(block_height, to_timestamp(block_time), amount, txid, txindex))
def insert_issuance(self, block_height, block_time, asset_id, amount, txid,
txindex, token, tokenamount):
self.conn.execute(
"INSERT INTO issuances VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(block_height, to_timestamp(block_time), asset_id, amount, txid,
txindex, token, tokenamount))
def log(file_handle, message):
"""
Very simple logging function, used by output class.
"""
file_handle.write("[%s] %s\n" % (to_timestamp(datetime.now()), message))
def log(file_handle, message):
"""
Very simple logging function, used by output class.
"""
file_handle.write("[%s] %s\n" % (to_timestamp(datetime.now()), message))
def filter_iovs_by_fcsr(self, upload_session_id):
"""
Ask for the server for the FCSR based on the synchronization type of the source Tag.
Then, modify the IOVs (possibly remove some) based on the FCSR we received.
This is useful in the case that most IOVs have different payloads, and our FCSR is close to the end of the range the IOVs cover.
"""
self._outputter.write(
"Getting the First Condition Safe Run for the current sync type.")
fcsr_data = self.get_fcsr_from_server(upload_session_id)
fcsr = fcsr_data["fcsr"]
fcsr_changed = fcsr_data["fcsr_changed"]
new_sync = fcsr_data["new_sync"]
if fcsr_changed:
self._outputter.write(
"Synchronization '%s' given was changed to '%s' to match destination Tag."
% (self.data_to_send["fcsr_filter"], new_sync))
self._outputter.write("Synchronization '%s' gave FCSR %d for FCSR Filtering."\
% (self.data_to_send["fcsr_filter"], friendly_since(self.data_to_send["inputTagData"]["time_type"], fcsr)))
"""
There may be cases where this assumption is not correct (that we can reassign since if fcsr > since)
Only set since to fcsr from server if the fcsr is further along than the user is trying to upload to
Note: this applies to run, lumi and timestamp run_types.
"""
# if the fcsr is above the since given by the user, we need to set the user since to the fcsr
if fcsr > self.data_to_send["since"]:
# check if we're uploading to offline sync - if so, then user since must be >= fcsr, so we should report an error
if self.data_to_send["fcsr_filter"].lower() == "offline":
self._outputter.write(
"If you're uploading to offline, you can't upload to a since < FCSR.\nNo upload has been processed."
)
self.exit_upload()
self.data_to_send["since"] = fcsr
self._outputter.write("Final FCSR after comparison with FCSR received from server is %d."\
% friendly_since(self.data_to_send["inputTagData"]["time_type"], int(self.data_to_send["since"])))
"""
Post validation processing assuming destination since is now valid.
Because we don't have an sqlite database to query (everything's in a dictionary),
we have to go through the IOVs manually find the greatest since that's less than
the destination since.
Purpose of this algorithm: move any IOV sinces that we can use up to the fcsr without leaving a hole in the Conditions coverage
"""
max_since_below_dest = self.data_to_send["iovs"][0]["since"]
for (i, iov) in enumerate(self.data_to_send["iovs"]):
if self.data_to_send["iovs"][i]["since"] <= self.data_to_send[
"since"] and self.data_to_send["iovs"][i][
"since"] > max_since_below_dest:
max_since_below_dest = self.data_to_send["iovs"][i]["since"]
# only select iovs that have sinces >= max_since_below_dest
# and then shift any IOVs left to the destination since
self.data_to_send["iovs"] = filter(
lambda iov: iov["since"] >= max_since_below_dest,
self.data_to_send["iovs"])
for (i, iov) in enumerate(self.data_to_send["iovs"]):
if self.data_to_send["iovs"][i]["since"] < self.data_to_send[
"since"]:
self.data_to_send["iovs"][i]["since"] = self.data_to_send[
"since"]
# modify insertion_time of iovs
new_time = to_timestamp(datetime.now())
for (i, iov) in enumerate(self.data_to_send["iovs"]):
self.data_to_send["iovs"][i]["insertion_time"] = new_time
def to_array(self):
return [self.name, self.time_type, self.object_type, self.synchronization, to_timestamp(self.insertion_time), self.description]
def get_time(x):
return to_timestamp(dt.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S'))
def filter_iovs_by_fcsr(self, upload_session_id):
"""
Ask for the server for the FCSR based on the synchronization type of the source Tag.
Then, modify the IOVs (possibly remove some) based on the FCSR we received.
This is useful in the case that most IOVs have different payloads, and our FCSR is close to the end of the range the IOVs cover.
"""
self._outputter.write("Getting the First Condition Safe Run for the current sync type.")
fcsr_data = self.get_fcsr_from_server(upload_session_id)
fcsr = fcsr_data["fcsr"]
fcsr_changed = fcsr_data["fcsr_changed"]
new_sync = fcsr_data["new_sync"]
if fcsr_changed:
self._outputter.write("Synchronization '%s' given was changed to '%s' to match destination Tag." % (self.data_to_send["fcsr_filter"], new_sync))
self._outputter.write("Synchronization '%s' gave FCSR %d for FCSR Filtering."\
% (self.data_to_send["fcsr_filter"], friendly_since(self.data_to_send["inputTagData"]["time_type"], fcsr)))
"""
There may be cases where this assumption is not correct (that we can reassign since if fcsr > since)
Only set since to fcsr from server if the fcsr is further along than the user is trying to upload to
Note: this applies to run, lumi and timestamp run_types.
"""
# if the fcsr is above the since given by the user, we need to set the user since to the fcsr
if fcsr > self.data_to_send["since"]:
# check if we're uploading to offline sync - if so, then user since must be >= fcsr, so we should report an error
if self.data_to_send["fcsr_filter"].lower() == "offline":
self._outputter.write("If you're uploading to offline, you can't upload to a since < FCSR.\nNo upload has been processed.")
self.exit_upload()
self.data_to_send["since"] = fcsr
self._outputter.write("Final FCSR after comparison with FCSR received from server is %d."\
% friendly_since(self.data_to_send["inputTagData"]["time_type"], int(self.data_to_send["since"])))
"""
Post validation processing assuming destination since is now valid.
Because we don't have an sqlite database to query (everything's in a dictionary),
we have to go through the IOVs manually find the greatest since that's less than
the destination since.
Purpose of this algorithm: move any IOV sinces that we can use up to the fcsr without leaving a hole in the Conditions coverage
"""
max_since_below_dest = self.data_to_send["iovs"][0]["since"]
for (i, iov) in enumerate(self.data_to_send["iovs"]):
if self.data_to_send["iovs"][i]["since"] <= self.data_to_send["since"] and self.data_to_send["iovs"][i]["since"] > max_since_below_dest:
max_since_below_dest = self.data_to_send["iovs"][i]["since"]
# only select iovs that have sinces >= max_since_below_dest
# and then shift any IOVs left to the destination since
self.data_to_send["iovs"] = [iov for iov in self.data_to_send["iovs"] if iov["since"] >= max_since_below_dest]
for (i, iov) in enumerate(self.data_to_send["iovs"]):
if self.data_to_send["iovs"][i]["since"] < self.data_to_send["since"]:
self.data_to_send["iovs"][i]["since"] = self.data_to_send["since"]
# modify insertion_time of iovs
new_time = to_timestamp(datetime.now())
for (i, iov) in enumerate(self.data_to_send["iovs"]):
self.data_to_send["iovs"][i]["insertion_time"] = new_time
def to_array(self):
return [self.name, self.release, to_timestamp(self.insertion_time), to_timestamp(self.snapshot_time), self.description]
def __init__(self, metadata_source=None, debug=False, verbose=False, testing=False, server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", **kwargs):
"""
Upload constructor:
Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded.
Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used.
Note: default value of service_url should be changed for production.
"""
# set private variables
self._debug = debug
self._verbose = verbose
self._testing = testing
# initialise server-side log data as empty string - will be replaced when we get a response back from the server
self._log_data = ""
self._SERVICE_URL = server
self.upload_session_id = None
# set up client-side log file
self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id()
self._handle = open(self.upload_log_file_name, "a")
# set up client-side logging object
self._outputter = output(verbose=verbose, log_handle=self._handle)
self._outputter.write("Using server instance at '%s'." % self._SERVICE_URL)
# expect a CondDBFW data_source object for metadata_source
if metadata_source == None:
# no upload metadat has been given - we cannot continue with the upload
self.exit_upload("A source of metadata must be given so CondDBFW knows how to upload conditions.")
else:
# set up global metadata source variable
self.metadata_source = metadata_source.data()
# check for the destination tag
# this is required whatever type of upload we're performing
if self.metadata_source.get("destinationTags") == None:
self.exit_upload("No destination Tag was given.")
else:
if isinstance(self.metadata_source.get("destinationTags"), dict) and self.metadata_source.get("destinationTags").keys()[0] == None:
self.exit_upload("No destination Tag was given.")
# make sure a destination database was given
if self.metadata_source.get("destinationDatabase") == None:
self.exit_upload("No destination database was given.")
# get Conditions metadata
if self.metadata_source.get("sourceDB") == None and self.metadata_source.get("hashToUse") == None:
"""
If we have neither an sqlite file nor the command line data
"""
self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\
+ "\nSee --help for command line argument information.")
elif self.metadata_source.get("sourceDB") != None:
"""
We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source
We now extract the Tag and IOV data from SQLite. It is added to the dictionary for sending over HTTPs later.
"""
# make sure we have an input tag to look for in the source db
self.input_tag = metadata_source.data().get("inputTag")
if self.input_tag == None:
self.exit_upload("No input Tag name was given.")
# set empty dictionary to contain Tag and IOV data from SQLite
result_dictionary = {}
self.sqlite_file_name = self.metadata_source["sourceDB"]
if not(os.path.isfile(self.sqlite_file_name)):
self.exit_upload("SQLite file '%s' given doesn't exist." % self.sqlite_file_name)
sqlite_con = querying.connect("sqlite://%s" % os.path.abspath(self.sqlite_file_name))
self._outputter.write("Getting Tag and IOVs from SQLite database.")
# query for Tag, check for existence, then convert to dictionary
tag = sqlite_con.tag(name=self.input_tag)
if tag == None:
self.exit_upload("The source Tag '%s' you gave was not found in the SQLite file." % self.input_tag)
tag = tag.as_dicts(convert_timestamps=True)
# query for IOVs, check for existence, then convert to dictionaries
iovs = sqlite_con.iov(tag_name=self.input_tag)
if iovs == None:
self.exit_upload("No IOVs found in the SQLite file given for Tag '%s'." % self.input_tag)
iovs = iovs.as_dicts(convert_timestamps=True)
iovs = [iovs] if not isinstance(iovs, list) else iovs
"""
Finally, get the list of all Payload hashes of IOVs,
then compute the list of hashes for which there is no Payload for
this is used later to decide if we can continue the upload if the Payload was not found on the server.
"""
iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag)
if iovs_for_hashes.__class__ == data_sources.json_list:
hashes_of_iovs = iovs_for_hashes.get_members("payload_hash").data()
else:
hashes_of_iovs = [iovs_for_hashes.payload_hash]
self.hashes_with_no_local_payload = [payload_hash for payload_hash in hashes_of_iovs if sqlite_con.payload(hash=payload_hash) == None]
# close session open on SQLite database file
sqlite_con.close_session()
elif metadata_source.data().get("hashToUse") != None:
"""
Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments).
We now use Tag and IOV data from command line. It is added to the dictionary for sending over HTTPs later.
"""
# set empty dictionary to contain Tag and IOV data from command line
result_dictionary = {}
now = to_timestamp(datetime.now())
# tag dictionary will be taken from the server
# this does not require any authentication
tag = self.get_tag_dictionary()
self.check_response_for_error_key(tag)
iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\
"insertion_time" : now}]
# hashToUse cannot be stored locally (no sqlite file is given), so register it as not found
self.hashes_with_no_local_payload = [self.metadata_source["hashToUse"]]
# Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side
# if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0
if tag["time_type"] == "Run":
for (i, iov) in enumerate(iovs):
iovs[i]["since"] = iovs[i]["since"] << 32
result_dictionary = {"inputTagData" : tag, "iovs" : iovs}
# add command line arguments to dictionary
# remembering that metadata_source is a json_dict object
result_dictionary.update(metadata_source.data())
# store in instance variable
self.data_to_send = result_dictionary
# if the since doesn't exist, take the first since from the list of IOVs
if result_dictionary.get("since") == None:
result_dictionary["since"] = sorted(iovs, key=lambda iov : iov["since"])[0]["since"]
elif self.data_to_send["inputTagData"]["time_type"] == "Run":
# Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing
self.data_to_send["since"] = self.data_to_send["since"] << 32
"""
TODO - Settle on a single destination tag format.
"""
# look for deprecated metadata entries - give warnings
# Note - we only really support this format
try:
if isinstance(result_dictionary["destinationTags"], dict):
self._outputter.write("WARNING: Multiple destination tags in a single metadata source is deprecated.")
except Exception as e:
self._outputter.write("ERROR: %s" % str(e))
def to_array(self):
return [self.queue, self.tag, self.record, self.label, status_full_name(self.status), to_timestamp(self.time_submitted), to_timestamp(self.last_edited)]
def insert_missed_block(self, expected_block_time, functionary):
self.conn.execute("INSERT INTO missing_blocks VALUES (?, ?)",
(to_timestamp(expected_block_time), functionary))
def insert_downtime(self, resume_time, downtime):
self.conn.execute("INSERT INTO outages VALUES (?, ?)",
(to_timestamp(resume_time), downtime))